1#!/usr/bin/env python 

2# encoding: utf-8 

3""" 

4Documentation for marshallEngine can be found here: http://marshallEngine.readthedocs.org 

5 

6Usage: 

7 marshall init 

8 marshall clean [-s <pathToSettingsFile>] 

9 marshall import <survey> [<withInLastDay>] [-s <pathToSettingsFile>] 

10 marshall lightcurve <transientBucketId> [-s <pathToSettingsFile>] 

11 marshall refresh <transientBucketId> [-s <pathToSettingsFile>] 

12 

13Options: 

14 init setup the marshallEngine settings file for the first time 

15 clean preform cleanup tasks like updating transient summaries table 

16 import import data, images, lightcurves from a feeder survey 

17 refresh update the cached metadata for a given transient 

18 lightcurve generate a lightcurve for a transient in the marshall database 

19 transientBucketId the transient ID from the database 

20 survey name of survey to import [panstarrs|atlas|useradded] 

21 withInLastDay import transient detections from the last N days (Default 30) 

22 

23 -h, --help show this help message 

24 -v, --version show version 

25 -s, --settings <pathToSettingsFile> the settings file 

26""" 

27from __future__ import print_function 

28import sys 

29import os 

30os.environ['TERM'] = 'vt100' 

31import readline 

32import glob 

33import pickle 

34from docopt import docopt 

35from fundamentals import tools, times 

36from subprocess import Popen, PIPE, STDOUT 

37 

38 

39def tab_complete(text, state): 

40 return (glob.glob(text + '*') + [None])[state] 

41 

42 

43def main(arguments=None): 

44 """ 

45 *The main function used when `cl_utils.py` is run as a single script from the cl, or when installed as a cl command* 

46 """ 

47 # setup the command-line util settings 

48 su = tools( 

49 arguments=arguments, 

50 docString=__doc__, 

51 logLevel="WARNING", 

52 options_first=False, 

53 projectName="marshall", 

54 defaultSettingsFile=True 

55 ) 

56 arguments, settings, log, dbConn = su.setup() 

57 

58 # tab completion for raw_input 

59 readline.set_completer_delims(' \t\n;') 

60 readline.parse_and_bind("tab: complete") 

61 readline.set_completer(tab_complete) 

62 

63 # UNPACK REMAINING CL ARGUMENTS USING `EXEC` TO SETUP THE VARIABLE NAMES 

64 # AUTOMATICALLY 

65 a = {} 

66 for arg, val in list(arguments.items()): 

67 if arg[0] == "-": 

68 varname = arg.replace("-", "") + "Flag" 

69 else: 

70 varname = arg.replace("<", "").replace(">", "") 

71 a[varname] = val 

72 if arg == "--dbConn": 

73 dbConn = val 

74 a["dbConn"] = val 

75 log.debug('%s = %s' % (varname, val,)) 

76 

77 ## START LOGGING ## 

78 startTime = times.get_now_sql_datetime() 

79 log.info( 

80 '--- STARTING TO RUN THE cl_utils.py AT %s' % 

81 (startTime,)) 

82 

83 init = a["init"] 

84 clean = a["clean"] 

85 iimport = a["import"] 

86 lightcurve = a["lightcurve"] 

87 transientBucketId = a["transientBucketId"] 

88 survey = a["survey"] 

89 withInLastDay = a["withInLastDay"] 

90 settingsFlag = a["settingsFlag"] 

91 

92 # set options interactively if user requests 

93 if "interactiveFlag" in a and a["interactiveFlag"]: 

94 

95 # load previous settings 

96 moduleDirectory = os.path.dirname(__file__) + "/resources" 

97 pathToPickleFile = "%(moduleDirectory)s/previousSettings.p" % locals() 

98 try: 

99 with open(pathToPickleFile): 

100 pass 

101 previousSettingsExist = True 

102 except: 

103 previousSettingsExist = False 

104 previousSettings = {} 

105 if previousSettingsExist: 

106 previousSettings = pickle.load(open(pathToPickleFile, "rb")) 

107 

108 # x-raw-input 

109 # x-boolean-raw-input 

110 # x-raw-input-with-default-value-from-previous-settings 

111 

112 # save the most recently used requests 

113 pickleMeObjects = [] 

114 pickleMe = {} 

115 theseLocals = locals() 

116 for k in pickleMeObjects: 

117 pickleMe[k] = theseLocals[k] 

118 pickle.dump(pickleMe, open(pathToPickleFile, "wb")) 

119 

120 if a["init"]: 

121 from os.path import expanduser 

122 home = expanduser("~") 

123 filepath = home + "/.config/marshallEngine/marshallEngine.yaml" 

124 try: 

125 cmd = """open %(filepath)s""" % locals() 

126 p = Popen(cmd, stdout=PIPE, stderr=PIPE, shell=True) 

127 except: 

128 pass 

129 try: 

130 cmd = """start %(filepath)s""" % locals() 

131 p = Popen(cmd, stdout=PIPE, stderr=PIPE, shell=True) 

132 except: 

133 pass 

134 return 

135 

136 # CALL FUNCTIONS/OBJECTS 

137 # DEFAULT VALUES 

138 if not withInLastDay: 

139 withInLastDay = 30 

140 

141 # CALL FUNCTIONS/OBJECTS 

142 if clean: 

143 # RESCUE ORPHANED TRANSIENTS - NO MASTER ID FLAG 

144 print("running stored procedures ...") 

145 from fundamentals.mysql import writequery 

146 

147 procedureNames = [ 

148 "update_transients_with_no_masteridflag()", 

149 "insert_new_transients_into_transientbucketsummaries()", 

150 "resurrect_objects()", 

151 "update_sherlock_xmatch_counts()", 

152 "update_inbox_auto_archiver()", 

153 "update_transient_akas(0)" 

154 ] 

155 

156 # CALL EACH PROCEDURE 

157 for p in procedureNames: 

158 sqlQuery = "CALL %(p)s;" % locals() 

159 print(f'Calling the `{p}` MySQL procedure') 

160 writequery( 

161 log=log, 

162 sqlQuery=sqlQuery, 

163 dbConn=dbConn, 

164 ) 

165 

166 # UPDATE THE TRANSIENT BUCKET SUMMARY TABLE IN THE MARSHALL DATABASE 

167 from marshallEngine.housekeeping import update_transient_summaries 

168 updater = update_transient_summaries( 

169 log=log, 

170 settings=settings, 

171 dbConn=dbConn 

172 ).update() 

173 

174 if iimport: 

175 if survey.lower() == "panstarrs": 

176 from marshallEngine.feeders.panstarrs.data import data 

177 from marshallEngine.feeders.panstarrs import images 

178 if survey.lower() == "atlas": 

179 from marshallEngine.feeders.atlas.data import data 

180 from marshallEngine.feeders.atlas import images 

181 if survey.lower() == "useradded": 

182 from marshallEngine.feeders.useradded.data import data 

183 from marshallEngine.feeders.useradded import images 

184 if survey.lower() == "tns": 

185 from marshallEngine.feeders.tns.data import data 

186 from marshallEngine.feeders.tns import images 

187 if survey.lower() == "ztf": 

188 from marshallEngine.feeders.ztf.data import data 

189 from marshallEngine.feeders.ztf import images 

190 if survey.lower() == "atels" or survey.lower() == "atel": 

191 from marshallEngine.feeders.atels.data import data 

192 from marshallEngine.feeders.atels import images 

193 ingester = data( 

194 log=log, 

195 settings=settings, 

196 dbConn=dbConn 

197 ).ingest(withinLastDays=withInLastDay) 

198 cacher = images( 

199 log=log, 

200 settings=settings, 

201 dbConn=dbConn 

202 ).cache(limit=3000) 

203 

204 from marshallEngine.services import panstarrs_location_stamps 

205 ps_stamp = panstarrs_location_stamps( 

206 log=log, 

207 settings=settings, 

208 dbConn=dbConn 

209 ).get() 

210 

211 if lightcurve: 

212 from marshallEngine.lightcurves import marshall_lightcurves 

213 lc = marshall_lightcurves( 

214 log=log, 

215 dbConn=dbConn, 

216 settings=settings, 

217 transientBucketIds=transientBucketId 

218 ) 

219 filepath = lc.plot() 

220 print("The lightcurve plot for transient %(transientBucketId)s can be found here: %(filepath)s" % locals()) 

221 

222 if a["refresh"]: 

223 from marshallEngine.housekeeping import update_transient_summaries 

224 updater = update_transient_summaries( 

225 log=log, 

226 settings=settings, 

227 dbConn=dbConn, 

228 transientBucketId=transientBucketId 

229 ).update() 

230 

231 if "dbConn" in locals() and dbConn: 

232 dbConn.commit() 

233 dbConn.close() 

234 ## FINISH LOGGING ## 

235 endTime = times.get_now_sql_datetime() 

236 runningTime = times.calculate_time_difference(startTime, endTime) 

237 log.info('-- FINISHED ATTEMPT TO RUN THE cl_utils.py AT %s (RUNTIME: %s) --' % 

238 (endTime, runningTime, )) 

239 

240 return 

241 

242if __name__ == '__main__': 

243 main()