; file: sdo_orderjsoc.pro ; init: Feb 1 2016 Rob Rutten Deil ; last: Jul 19 2021 Rob Rutten Deil ; todo: ?? concatenate record sets into only 1 order and email = less wait ; note: JSOC works in TAI, HMI also; TAI=UT+37s (36s before 2017) ; JSOC seems to use t_rec for file selection, not t_obs ; t_obs-t_rec = plm 7.58s@304, 11.3s @171, 18.2s@1700, Os@HMI ;+ pro sdo_orderjsoc,datetimestart,duration,xsol,ysol,email,name,$ requestidents,requestsizes,$ xsize=xsize,ysize=ysize,wavs=wavs,cadence=cadence,notrack=notrack,$ waitduration=waitduration ; NAME: ; sdo_orderjsoc ; ; PURPOSE: ; order SDO im_patch cutouts, emulating clicking on JSOC Exportdata site ; ; METHOD: ; use SolarSoft sock_list.pro by D. Zarro; find URL from Exportdata check ; ; INPUTS: ; datetimestart (UTC): start in fixed-format string as '2015.09.27_07:00' ; duration: minutes, e.g., 60 (no quotes, integer) ; xsol, ysol: solar (X,Y) in arcsec of desired cutout center at start ; email: requester's (must be registered at JSOC) ; name: requester's name ; ; OUTPUTS ; requestidents: string array with JSOC request identifiers ; requestsizes: float array with JSOC size estimates ; ; OPTIONAL KEYWORD INPUTS: ; xsize, ysize: cutout field of view in arcsec (default: 150x150 = SST) ; wavs: desired SDO channels in 1D string array ; (default: AIA all EUV and UV; HMI magnetogram, continuum) ; cadence: string, e.g. '5m' ; default = 'fast' = fastest each: EUV 12s, UV 24s, HMI 45s ; notrack=1/0: switch off cutout rotation tracking by JSOC, default 0 ; waitduration: wait in seconds when JSOC busy with previous (default 30) ; ; EXAMPLE: ; below program end below ; ; RESTRICTIONS: ; requires IDL 6.4 or later ; repeat requests with the same parameters are not honored (smart Phil); ; you then get the 'data ordered' statements immediately, no email ; ; HISTORY: ; Feb 1 2016 RR: start = suggestion by Peter Young during ISSI ; Feb 26 2016 RR: cadence (to order drift sets) ; Apr 8 2016 RR: notrack, ntostr>trim, get idents and sizes ; all suggested by Peter Young ; Feb 7 2020 RR: added waits because JSOC now forbids rapid fire ;- ; answer no-parameter query if (n_params() lt 8) then begin sp,order_jsoc return endif ; default keywords if (n_elements(xsize) eq 0) then xsize=150. ;RR good for SST if (n_elements(ysize) eq 0) then ysize=150. if (n_elements(wavs) eq 0) then $ wavs=['94','131','171','193','211','304','335','1600','1700',$ 'magnetogram','continuum'] if (n_elements(cadence) eq 0) then cadence='fast' if (n_elements(notrack) eq 0) then notrack=0 ;RR JSOC request parameter t=0 means tracking is on so call it "notrack" if (n_elements(waitduration) eq 0) then waitduration=30 ; check user ident when email is mine if (email eq 'robenrietjerutten@gmail.com') then begin spawn,'whoami > sdo_getdata_reqester' readcol,'sdo_getdata_reqester',username,format='A',/silent spawn,'rm -f sdo_getdata_reqester' if (username ne 'rutten' and username ne 'rutte101') then begin print,' ##### sdo_orderjsoc abort: email = RJR but username not RJR' print,' ##### specify your own email address in calling this program!' retall endif endif ; define cadences if (cadence ne 'fast') then $ if (strpos(cadence,'s') gt 0 and fix(cadence) lt 45) then begin cadence='fast' print,' ===== sdo_orderjsoc: cadence increased to minimum value' endif if (cadence eq 'fast') then begin cadenceeuv='12.00s' cadenceuv='24.00s' cadencehmi='45.0s' atpart='' endif else begin cadenceeuv=cadence cadenceuv=cadence cadencehmi=cadence atpart='@'+cadence endelse ; define AIA "channels" and HMI "series" (whatsinaname) sdowavs=['94','131','171','193','211','304','335',$ '1600','1700','4500',$ 'magnetogram','continuum','Dopplergram'] ndeswavs=n_elements(wavs) ; sizeratio = fraction of full 4Kx4K image size sizeratio=(xsize/0.6*ysize/0.6)/(4096.*4096) ; now do the requests in order smallest to largest (for response time) requestidents='none' requestsizes=0 nrepeat=0 ; define time strings the TAI way JSOC wants them for im_patch tstarttai=anytim2tai(datetimestart) tstartjsoc=datetimestart+':00_TAI' ;RR append TAI = 37 sec early durationjsoc=duration+1 ; needs extra minute tstoptai=tstarttai+60.*durationjsoc tstopjsoc=strmid(tstoptai,0,19)+'_TAI' ;RR append TAI strput,tstopjsoc,'_',10 ; ============ request HMI sequences (each separate) ===================== ; find desired HMI channels hmiwavs='0' for ideswav=0,ndeswavs-1 do begin for isdowav=10,12 do $ if (wavs[ideswav] eq sdowavs[isdowav]) then $ hmiwavs=[hmiwavs,sdowavs[isdowav]] endfor nhmi=n_elements(hmiwavs) ; loop over requests if (nhmi gt 1) then begin for ihmi=1,nhmi-1 do begin ; select HMI series if (hmiwavs[ihmi] eq 'magnetogram') then hmiser='hmi.M_45s' if (hmiwavs[ihmi] eq 'continuum') then hmiser='hmi.Ic_45s' if (hmiwavs[ihmi] eq 'Dopplergram') then hmiser='hmi.V_45s' ; define orderstring this HMI series orderstring='op=exp_request&ds='+hmiser+'['+tstartjsoc+'/'+trim(fix(durationjsoc))+'m'+atpart+']&sizeratio='+trim(sizeratio)+'&process=n=0|im_patch,t_start='+tstartjsoc+',t_stop='+tstopjsoc+',t='+trim(notrack)+',r=0,c=0,cadence='+cadencehmi+',locunits=arcsec,boxunits=arcsec,t_ref='+tstartjsoc+',x='+trim(xsol)+',y='+trim(ysol)+',width='+trim(xsize)+',height='+trim(ysize)+'&requestor='+name+'¬ify='+email+'&method=url-tar&filenamefmt='+hmiser+'.{T_REC:A}.{CAMERA}.{segment}&format=json&protocol=FITS,compress Rice&dbhost=hmidb2' ;RR inspect orderstring for comparison with manual from JSOC Exportdata ;; print,' ----- orderstring: '+orderstring ; order HMI with wait + repeat if JSOC still busy or gives error AGAINHMI: sock_list,$ 'http://jsoc.stanford.edu/cgi-bin/ajax/jsocextfetch?'+orderstring,output if (output eq '' or $ strmatch(output,'*pending export request*')) then begin print, ' HMI order waits on your still running JSOC order '+$ '(no parallel ordering)' wait,waitduration goto,AGAINHMI endif ; check for other error, repeat when the case quotepos=strsplit(output,'"') nquotes=n_elements(quotepos) if (nquotes lt 10) then begin print,' ===== sdo_orderjsoc wait: error JSOC HMI order '+$ hmiwavs[ihmi]+' giving output: '+output wait,waitduration goto,AGAINHMI endif else begin ; get order ident and size hmiident=strmid(output,quotepos[5],quotepos[6]-quotepos[5]-1) requestidents=[requestidents,hmiident] hmisize=float(strmid(output,quotepos[nquotes-1]+1)) requestsizes=[requestsizes,hmisize] endelse ; print confirmation print,' ===== HMI data ordered: '+hmiser+' = '+hmiwavs[ihmi] print,' JSOC identifier: '+hmiident print,' size estimate: '+trim(hmisize) ; check on repeat request ;;; checkhmi=duration*60./cadencehmi*sizeratio*4.E2 ;RR floats if (hmisize lt 20) then begin print,' ===== small size suggests repeat: '+$ ' no email or ident = earlier request?' nrepeat=nrepeat+1 endif endfor ; end loop over HMI diagnostics endif ; end HMI diagnostics ; ============ request AIA UV sequences ======================= ; find desired UV channels uvwavs=0 for ideswav=0,ndeswavs-1 do begin for isdowav=7,8 do $ if (wavs[ideswav] eq sdowavs[isdowav]) then $ uvwavs=[uvwavs,fix(sdowavs[isdowav])] endfor nuv=n_elements(uvwavs) if (nuv gt 1) then begin uvstring=trim(uvwavs[1]) for iuv=2,nuv-1 do uvstring=uvstring+','+trim(uvwavs[iuv]) ; define orderstring for UV channel data orderstring='op=exp_request&ds=aia.lev1_uv_24s['+tstartjsoc+'/'+trim(fix(durationjsoc))+'m'+atpart+']['+uvstring+']&sizeratio='+trim(sizeratio)+'&process=n=0|im_patch,t_start='+tstartjsoc+',t_stop='+tstopjsoc+',t='+trim(notrack)+',r=0,c=0,cadence='+cadenceuv+',locunits=arcsec,boxunits=arcsec,t_ref='+tstartjsoc+',x='+trim(xsol)+',y='+trim(ysol)+',width='+trim(xsize)+',height='+trim(ysize)+'&requestor='+name+'¬ify='+email+'&method=url-tar&filenamefmt=aia.lev1_uv_24s.{T_REC:A}.{WAVELNTH}.{segment}&format=json&protocol=FITS,compress Rice&dbhost=hmidb2' ;; [? QUALITY >= 0 ?]' suggest by Patrick, error ; order UV with wait + repeat if JSOC still busy AGAINUV: sock_list,$ 'http://jsoc.stanford.edu/cgi-bin/ajax/jsocextfetch?'+orderstring,output if (output eq '' or $ strmatch(output,'*pending export request*')) then begin print, ' UV order waits on your still running JSOC order' wait,waitduration goto, AGAINUV endif ; check for other error, repeat when the case quotepos=strsplit(output,'"') nquotes=n_elements(quotepos) if (nquotes lt 10) then begin print,' ===== sdo_orderjsoc wait: error JSOC AIA UV order' print,' giving output: '+output wait,waitduration goto,AGAINUV endif else begin ; get order ident and size uvident=strmid(output,quotepos[5],quotepos[6]-quotepos[5]-1) requestidents=[requestidents,uvident] uvsize=float(strmid(output,quotepos[nquotes-1]+1)) requestsizes=[requestsizes,uvsize] endelse ; print confirmation print,' ===== UV data ordered: '+uvstring print,' JSOC identifier: '+uvident print,' size estimate: '+trim(uvsize) ; check on repeat request ;;; checkuv=duration*60./cadenceuv*sizeratio*(nuv-1)*2.E2 ;RR integers if (uvsize lt 50) then begin print,' ##### small size suggests repeat: '+$ ' no email or ident = earlier request?' nrepeat=nrepeat+1 endif endif ; end UV channels ; ============ request AIA EUV sequences ======================== ; find desired EUV channels euvwavs=0 for ideswav=0,ndeswavs-1 do begin for isdowav=0,6 do $ if (wavs[ideswav] eq sdowavs[isdowav]) then $ euvwavs=[euvwavs,fix(sdowavs[isdowav])] endfor neuv=n_elements(euvwavs) if (neuv gt 1) then begin euvstring=trim(euvwavs[1]) for ieuv=2,neuv-1 do euvstring=euvstring+','+trim(euvwavs[ieuv]) ; define orderstring for EUV data orderstring='op=exp_request&ds=aia.lev1_euv_12s['+tstartjsoc+'/'+trim(fix(durationjsoc))+'m'+atpart+']['+euvstring+']&sizeratio='+trim(sizeratio)+'&process=n=0|im_patch,t_start='+tstartjsoc+',t_stop='+tstopjsoc+',t='+trim(notrack)+',r=0,c=0,cadence='+cadenceeuv+',locunits=arcsec,boxunits=arcsec,t_ref='+tstartjsoc+',x='+trim(xsol)+',y='+trim(ysol)+',width='+trim(xsize)+',height='+trim(ysize)+'&requestor='+name+'¬ify='+email+'&method=url-tar&filenamefmt=aia.lev1_euv_12s.{T_REC:A}.{WAVELNTH}.{segment}&format=json&protocol=FITS,compress Rice&dbhost=hmidb2' ; order with wait+repeat if JSOC still busy AGAINEUV: sock_list,$ 'http://jsoc.stanford.edu/cgi-bin/ajax/jsocextfetch?'+orderstring,output if (output eq '' or $ strmatch(output,'*pending export request*')) then begin print, ' EUV order waits on your still running JSOC order' wait,waitduration goto, AGAINEUV endif ; get order ident and size; test is primitive (too fixed, how better)? quotepos=strsplit(output,'"') nquotes=n_elements(quotepos) if (nquotes lt 10) then begin print,' ===== sdo_orderjsoc wait: error JSOC AIA EUV order' print,' giving output: '+output wait,waitduration goto,AGAINEUV endif else begin euvident=strmid(output,quotepos[5],quotepos[6]-quotepos[5]-1) requestidents=[requestidents,euvident] euvsize=float(strmid(output,quotepos[nquotes-1]+1)) requestsizes=[requestsizes,euvsize] endelse ; print confirmation print,' ===== EUV data ordered: '+euvstring print,' JSOC identifier: '+euvident print,' size estimate: '+trim(euvsize) ; check on repeat request ;;; checkeuv=duration*60./cadenceuv*sizeratio*(neuv-1)*2.E2 ;RR integers if (euvsize lt 100) then begin print,' ##### small size suggests repeat: '+$ ' no email or ident = earlier request?' nrepeat=nrepeat+1 endif endif ; end of EUV channels ; end of JSOC ordering: strip array initiators nrequests=n_elements(requestidents) if (nrequests-1-nrepeat lt 1) then begin print, ' ##### sdo_orderjsoc.pro did not order anything' endif requestidents=requestidents[1:nrequests-1] requestsizes=requestsizes[1:nrequests-1] ; print mail expectation print,' ===== you should get '+trim(nrequests-1-nrepeat)+' new JSOC emails' print,' in each: rightclick on link, then rightclick to download' print,' or use sdo_getjsoc.pro (part of sdo_getdata_rr.pro)' end ; ================ tests per IDLWAVE Hyper-C ======================== ; standard test for SST alignment cd,'/home/rutten/data/SDO/2014-06-14-small' sdo_orderjsoc,'2014.06.14_09:00',7,-432,-312,$ 'robenrietjerutten@gmail.com','Rob Rutten',$ ;; wavs=['magnetogram'],$ ;; wavs=['continuum'],$ requestidents,requestsizes end