1#!/usr/bin/env python 

2# encoding: utf-8 

3""" 

4Documentation for sherlock can be found here: http://sherlock.readthedocs.org 

5 

6.. todo :: 

7 

8 - docuument cl_utils module 

9 - tidy usage text 

10 

11Usage: 

12 sherlock init 

13 sherlock info [-s <pathToSettingsFile>] 

14 sherlock [-NA] dbmatch [--update] [-s <pathToSettingsFile>] 

15 sherlock [-bN] match -- <ra> <dec> [<pathToSettingsFile>]  

16 sherlock clean [-s <pathToSettingsFile>] 

17 sherlock wiki [-s <pathToSettingsFile>] 

18 sherlock import ned <ra> <dec> <radiusArcsec> [-s <pathToSettingsFile>] 

19 sherlock import cat <cat_name> <pathToDataFile> <cat_version> [-s <pathToSettingsFile>] 

20 sherlock import stream <stream_name> [-s <pathToSettingsFile>] 

21 

22Options: 

23 init setup the sherlock settings file for the first time 

24 match XXXX 

25 dbmatch database match 

26 clean XXXX 

27 wiki XXXX 

28 import XXXX 

29 ned use the online NED database as the source catalogue 

30 cat import a static catalogue into the sherlock-catalogues database 

31 stream download/stream new data from a give source catalogue into the sherlock sherlock-catalogues database 

32 info print an overview of the current catalogues, views and streams in the sherlock database ready for crossmatching 

33 

34 ra the right-ascension coordinate with which to perform a conesearch (sexegesimal or decimal degrees) 

35 dec the declination coordinate with which to perform a conesearch (sexegesimal or decimal degrees) 

36 radiusArcsec radius in arcsec of the footprint to download from the online NED database 

37 cat_name name of the catalogue being imported (veron|ned_d)  

38 stream_name name of the stream to import into the sherlock-catalogues database (ifs) 

39 

40 -N, --skipNedUpdate do not update the NED database before classification 

41 -A, --skipMagUpdate do not update the peak magnitudes and human readable text annotations of objects (can eat up some time) 

42 -h, --help show this help message 

43 -s, --settings the settings file 

44 -b, --verbose print more details to stdout 

45 -u, --update update the transient database with new classifications and crossmatches 

46 -v, --version print the version of sherlock 

47""" 

48from __future__ import print_function 

49from __future__ import absolute_import 

50import sys 

51import os 

52os.environ['TERM'] = 'vt100' 

53import readline 

54import glob 

55import pickle 

56from docopt import docopt 

57from fundamentals import tools, times 

58from subprocess import Popen, PIPE, STDOUT 

59from fundamentals.renderer import list_of_dictionaries 

60from .database_cleaner import database_cleaner 

61from .commonutils import update_wiki_pages 

62from sherlock.imports import veron as veronImporter 

63from sherlock.imports import ifs as ifsImporter 

64from sherlock.imports import ned_d as nedImporter 

65from sherlock.imports import ned as nedStreamImporter 

66from sherlock.commonutils import update_wiki_pages 

67from sherlock import transient_classifier 

68 

69 

70def tab_complete(text, state): 

71 return (glob.glob(text + '*') + [None])[state] 

72 

73 

74def main(arguments=None): 

75 """ 

76 *The main function used when `cl_utils.py` is run as a single script from the cl, or when installed as a cl command* 

77 """ 

78 # setup the command-line util settings 

79 su = tools( 

80 arguments=arguments, 

81 docString=__doc__, 

82 logLevel="WARNING", 

83 options_first=False, 

84 distributionName="qub-sherlock", 

85 projectName="sherlock", 

86 defaultSettingsFile=True 

87 ) 

88 arguments, settings, log, dbConn = su.setup() 

89 

90 # tab completion for raw_input 

91 readline.set_completer_delims(' \t\n;') 

92 readline.parse_and_bind("tab: complete") 

93 readline.set_completer(tab_complete) 

94 

95 # UNPACK REMAINING CL ARGUMENTS USING `EXEC` TO SETUP THE VARIABLE NAMES 

96 # AUTOMATICALLY 

97 a = {} 

98 for arg, val in list(arguments.items()): 

99 if arg[0] == "-": 

100 varname = arg.replace("-", "") + "Flag" 

101 else: 

102 varname = arg.replace("<", "").replace(">", "") 

103 a[varname] = val 

104 if arg == "--dbConn": 

105 dbConn = val 

106 a["dbConn"] = val 

107 log.debug('%s = %s' % (varname, val,)) 

108 

109 ## START LOGGING ## 

110 startTime = times.get_now_sql_datetime() 

111 log.info( 

112 '--- STARTING TO RUN THE cl_utils.py AT %s' % 

113 (startTime,)) 

114 

115 # set options interactively if user requests 

116 if "interactiveFlag" in a and a["interactiveFlag"]: 

117 

118 # load previous settings 

119 moduleDirectory = os.path.dirname(__file__) + "/resources" 

120 pathToPickleFile = "%(moduleDirectory)s/previousSettings.p" % locals() 

121 try: 

122 with open(pathToPickleFile): 

123 pass 

124 previousSettingsExist = True 

125 except: 

126 previousSettingsExist = False 

127 previousSettings = {} 

128 if previousSettingsExist: 

129 previousSettings = pickle.load(open(pathToPickleFile, "rb")) 

130 

131 # x-raw-input 

132 # x-boolean-raw-input 

133 # x-raw-input-with-default-value-from-previous-settings 

134 

135 # save the most recently used requests 

136 pickleMeObjects = [] 

137 pickleMe = {} 

138 theseLocals = locals() 

139 for k in pickleMeObjects: 

140 pickleMe[k] = theseLocals[k] 

141 pickle.dump(pickleMe, open(pathToPickleFile, "wb")) 

142 

143 if a["init"]: 

144 from os.path import expanduser 

145 home = expanduser("~") 

146 filepath = home + "/.config/sherlock/sherlock.yaml" 

147 try: 

148 cmd = """open %(filepath)s""" % locals() 

149 p = Popen(cmd, stdout=PIPE, stderr=PIPE, shell=True) 

150 except: 

151 pass 

152 try: 

153 cmd = """start %(filepath)s""" % locals() 

154 p = Popen(cmd, stdout=PIPE, stderr=PIPE, shell=True) 

155 except: 

156 pass 

157 return 

158 

159 init = a["init"] 

160 match = a["match"] 

161 dbmatch = a["dbmatch"] 

162 clean = a["clean"] 

163 wiki = a["wiki"] 

164 iimport = a["import"] 

165 ned = a["ned"] 

166 cat = a["cat"] 

167 stream = a["stream"] 

168 info = a["info"] 

169 ra = a["ra"] 

170 dec = a["dec"] 

171 radiusArcsec = a["radiusArcsec"] 

172 cat_name = a["cat_name"] 

173 stream_name = a["stream_name"] 

174 skipNedUpdateFlag = a["skipNedUpdateFlag"] 

175 skipMagUpdateFlag = a["skipMagUpdateFlag"] 

176 settingsFlag = a["settingsFlag"] 

177 verboseFlag = a["verboseFlag"] 

178 updateFlag = a["updateFlag"] 

179 

180 # CALL FUNCTIONS/OBJECTS 

181 if match or dbmatch: 

182 if verboseFlag: 

183 verbose = 2 

184 else: 

185 verbose = 1 

186 

187 if skipNedUpdateFlag: 

188 updateNed = False 

189 else: 

190 updateNed = True 

191 

192 if skipMagUpdateFlag: 

193 updatePeakMags = False 

194 else: 

195 updatePeakMags = True 

196 

197 classifier = transient_classifier.transient_classifier( 

198 log=log, 

199 settings=settings, 

200 ra=ra, 

201 dec=dec, 

202 name=False, 

203 verbose=verbose, 

204 update=updateFlag, 

205 updateNed=updateNed, 

206 updatePeakMags=updatePeakMags 

207 ) 

208 classifier.classify() 

209 

210 if clean: 

211 cleaner = database_cleaner( 

212 log=log, 

213 settings=settings 

214 ) 

215 cleaner.clean() 

216 if wiki: 

217 updateWiki = update_wiki_pages( 

218 log=log, 

219 settings=settings 

220 ) 

221 updateWiki.update() 

222 

223 if iimport and ned: 

224 ned = nedStreamImporter( 

225 log=log, 

226 settings=settings, 

227 coordinateList=["%(ra)s %(dec)s" % locals()], 

228 radiusArcsec=radiusArcsec 

229 ) 

230 ned.ingest() 

231 if iimport and cat: 

232 

233 if cat_name == "veron": 

234 catalogue = veronImporter( 

235 log=log, 

236 settings=settings, 

237 pathToDataFile=pathToDataFile, 

238 version=cat_version, 

239 catalogueName=cat_name 

240 ) 

241 catalogue.ingest() 

242 

243 if "ned_d" in cat_name: 

244 catalogue = nedImporter( 

245 log=log, 

246 settings=settings, 

247 pathToDataFile=pathToDataFile, 

248 version=cat_version, 

249 catalogueName=cat_name 

250 ) 

251 catalogue.ingest() 

252 if iimport and stream: 

253 if "ifs" in stream_name: 

254 stream = ifsImporter( 

255 log=log, 

256 settings=settings 

257 ) 

258 stream.ingest() 

259 if not init and not match and not clean and not wiki and not iimport and ra: 

260 

261 classifier = transient_classifier.transient_classifier( 

262 log=log, 

263 settings=settings, 

264 ra=ra, 

265 dec=dec, 

266 name=False, 

267 verbose=verboseFlag 

268 ) 

269 classifier.classify() 

270 

271 if info: 

272 print("sherlock-catalogues") 

273 wiki = update_wiki_pages( 

274 log=log, 

275 settings=settings 

276 ) 

277 table = list(wiki._get_table_infos(trimmed=True)) 

278 

279 dataSet = list_of_dictionaries( 

280 log=log, 

281 listOfDictionaries=table 

282 ) 

283 tableData = dataSet.reST(filepath=None) 

284 print(tableData) 

285 print() 

286 

287 print("Crossmatch Streams") 

288 table = list(wiki._get_stream_view_infos(trimmed=True)) 

289 dataSet = list_of_dictionaries( 

290 log=log, 

291 listOfDictionaries=table 

292 ) 

293 tableData = dataSet.reST(filepath=None) 

294 print(tableData) 

295 print() 

296 

297 print("Views on Catalogues and Streams") 

298 

299 table = list(wiki._get_view_infos(trimmed=True)) 

300 dataSet = list_of_dictionaries( 

301 log=log, 

302 listOfDictionaries=table 

303 ) 

304 tableData = dataSet.reST(filepath=None) 

305 print(tableData) 

306 

307 if "dbConn" in locals() and dbConn: 

308 dbConn.commit() 

309 dbConn.close() 

310 ## FINISH LOGGING ## 

311 endTime = times.get_now_sql_datetime() 

312 runningTime = times.calculate_time_difference(startTime, endTime) 

313 log.info('-- FINISHED ATTEMPT TO RUN THE cl_utils.py AT %s (RUNTIME: %s) --' % 

314 (endTime, runningTime, )) 

315 

316 return 

317 

318if __name__ == '__main__': 

319 main()