1#!/usr/local/bin/python 

2# encoding: utf-8 

3""" 

4*Update the transient summaries table in the marshall database with the top-level metadata* 

5 

6:Author: 

7 David Young 

8""" 

9from __future__ import print_function 

10from builtins import zip 

11from builtins import str 

12from builtins import object 

13import sys 

14import os 

15os.environ['TERM'] = 'vt100' 

16from fundamentals import tools 

17from fundamentals.mysql import writequery, readquery 

18from astropy import units as u 

19from astropy.coordinates import SkyCoord 

20from fundamentals.mysql import insert_list_of_dictionaries_into_database_tables 

21from astrocalc.distances import converter 

22import numpy as np 

23from HMpTy.mysql import add_htm_ids_to_mysql_database_table 

24 

25 

26class update_transient_summaries(object): 

27 """ 

28 *Update the transient summaries table in the marshall database* 

29 

30 **Key Arguments** 

31 

32 - ``log`` -- logger 

33 - ``settings`` -- the settings dictionary 

34 - ``dbConn`` -- the marshall database connection 

35 - ``transientBucketId`` -- a single transientBucketId to update transientBucketId. Default *False* (i.e. update all) 

36 

37 

38 **Usage** 

39 

40 To setup your logger, settings and database connections, please use the ``fundamentals`` package (`see tutorial here <http://fundamentals.readthedocs.io/en/latest/#tutorial>`_). 

41 

42 To initiate a update_transient_summaries object, use the following: 

43 

44 .. todo:: 

45 

46 - add a tutorial about ``update_transient_summaries`` to documentation 

47 

48 ```python 

49 from marshallEngine.housekeeping import update_transient_summaries 

50 updater = update_transient_summaries( 

51 log=log, 

52 settings=settings, 

53 dbConn=dbConn, 

54 transientBucketId=False 

55 ).update() 

56 ``` 

57 

58 """ 

59 # Initialisation 

60 

61 def __init__( 

62 self, 

63 log, 

64 dbConn, 

65 settings=False, 

66 transientBucketId=False 

67 ): 

68 self.log = log 

69 log.debug("instansiating a new 'update_transient_summaries' object") 

70 self.settings = settings 

71 self.transientBucketIds = [] 

72 self.transientBucketId = transientBucketId 

73 

74 # xt-self-arg-tmpx 

75 

76 self.dbConn = dbConn 

77 

78 if self.transientBucketId: 

79 print( 

80 "updating transient summaries table for %(transientBucketId)s" % locals()) 

81 # UPDATE TRANSIENT BUCKET SUMMARIES (IN MYSQL) 

82 sqlQuery = "call update_single_transientbucket_summary(%(transientBucketId)s)" % locals( 

83 ) 

84 writequery( 

85 log=self.log, 

86 sqlQuery=sqlQuery, 

87 dbConn=self.dbConn 

88 ) 

89 else: 

90 # UPDATE OBSERVATION DATES FROM MJDs 

91 sqlQuery = "call update_transientbucket_observation_dates()" 

92 writequery( 

93 log=self.log, 

94 sqlQuery=sqlQuery, 

95 dbConn=self.dbConn 

96 ) 

97 

98 print("updating transient summaries table") 

99 # UPDATE TRANSIENT BUCKET SUMMARIES (IN MYSQL) 

100 sqlQuery = "call update_transientbucketsummaries()" 

101 writequery( 

102 log=self.log, 

103 sqlQuery=sqlQuery, 

104 dbConn=self.dbConn 

105 ) 

106 

107 print("updating sherlock crossmatches table") 

108 sqlQuery = "call update_sherlock_crossmatches()" 

109 writequery( 

110 log=self.log, 

111 sqlQuery=sqlQuery, 

112 dbConn=self.dbConn 

113 ) 

114 

115 return None 

116 

117 def update(self): 

118 """ 

119 *Update the transient summaries table in the marshall database* 

120 

121 **Return** 

122 

123 - ``update_transient_summaries`` 

124 

125 

126 **Usage** 

127 

128 

129 

130 ```python 

131 from marshallEngine.housekeeping import update_transient_summaries 

132 updater = update_transient_summaries( 

133 log=log, 

134 settings=settings, 

135 dbConn=dbConn 

136 ).update() 

137 ``` 

138 """ 

139 self.log.debug('starting the ``get`` method') 

140 

141 self._add_galactic_coords() 

142 self._add_distances() 

143 

144 # UPDATE LIGHTCURVE PLOTS 

145 

146 from marshallEngine.lightcurves import marshall_lightcurves 

147 lc = marshall_lightcurves( 

148 log=self.log, 

149 dbConn=self.dbConn, 

150 settings=self.settings, 

151 transientBucketIds=self.transientBucketIds 

152 ) 

153 lc.plot() 

154 

155 transientBucketIds = ('","').join(str(x) 

156 for x in self.transientBucketIds) 

157 

158 # RESET UPDATE-NEEDED FLAG 

159 sqlQuery = """update transientBucketSummaries set updateNeeded = 0 where updateNeeded = 2 and transientBucketId in ("%(transientBucketIds)s")""" % locals( 

160 ) 

161 writequery( 

162 log=self.log, 

163 sqlQuery=sqlQuery, 

164 dbConn=self.dbConn 

165 ) 

166 

167 # ADD HTM IDs 

168 self._update_htm_columns() 

169 

170 self.log.debug('completed the ``get`` method') 

171 return None 

172 

173 def _add_galactic_coords( 

174 self): 

175 """*add galactic coordinates to the summary table* 

176 

177 **Usage** 

178 

179 ```python 

180 from marshallEngine.housekeeping import update_transient_summaries 

181 updater = update_transient_summaries( 

182 log=log, 

183 settings=settings, 

184 dbConn=dbConn 

185 ) 

186 updater._add_galactic_coords() 

187 ``` 

188 

189 """ 

190 self.log.debug('starting the ``_add_galactic_coords`` method') 

191 

192 # SELECT THE TRANSIENTS NEEDING UPDATED 

193 extra = "" 

194 if self.transientBucketId: 

195 thisId = self.transientBucketId 

196 extra = "and transientBucketId = %(thisId)s" % locals() 

197 

198 sqlQuery = u""" 

199 select raDeg, decDeg, transientBucketId from transientBucketSummaries where updateNeeded = 2 %(extra)s order by transientBucketId desc 

200 """ % locals() 

201 rows = readquery( 

202 log=self.log, 

203 sqlQuery=sqlQuery, 

204 dbConn=self.dbConn, 

205 quiet=False 

206 ) 

207 

208 total = len(rows) 

209 # STOP IF NOTHING TO UPDATE 

210 if not total: 

211 return 

212 

213 if total > 1000: 

214 print( 

215 """%(total)s transients need updated - updating the next 1000""" % locals()) 

216 rows = np.random.choice(rows, size=1000, replace=False, p=None) 

217 

218 # CREATE 3 LISTS - RA, DEC, ID 

219 raDeg = [] 

220 raDeg[:] = [r["raDeg"] * u.degree for r in rows] 

221 decDeg = [] 

222 decDeg[:] = [r["decDeg"] * u.degree for r in rows] 

223 ids = [] 

224 ids[:] = [r["transientBucketId"] for r in rows] 

225 self.transientBucketIds = ids 

226 

227 # DETERMINE GALACTIC COORDINATES 

228 c = SkyCoord(ra=raDeg, dec=decDeg, frame='icrs') 

229 l = c.galactic.l.deg 

230 b = c.galactic.b.deg 

231 

232 # CREATE A LIST OF DICTIONARIES 

233 dictList = [] 

234 for gl, gb, tid in zip(l, b, ids): 

235 dictList.append({ 

236 "glat": gb, 

237 "glon": gl, 

238 "transientBucketId": tid 

239 }) 

240 

241 if not len(dictList): 

242 return 

243 

244 # USE dbSettings TO ACTIVATE MULTIPROCESSING - INSERT LIST OF 

245 # DICTIONARIES INTO DATABASE 

246 insert_list_of_dictionaries_into_database_tables( 

247 dbConn=self.dbConn, 

248 log=self.log, 

249 dictList=dictList, 

250 dbTableName="transientBucketSummaries", 

251 dateModified=False, 

252 dateCreated=False, 

253 batchSize=2500, 

254 replace=True, 

255 dbSettings=self.settings["database settings"] 

256 ) 

257 

258 self.log.debug('completed the ``_add_galactic_coords`` method') 

259 return None 

260 

261 def _add_distances( 

262 self): 

263 """*Add a distance measurement from the best redshift if the transient does not already have a distance measurement* 

264 

265 **Usage** 

266 

267 ```python 

268 from marshallEngine.housekeeping import update_transient_summaries 

269 updater = update_transient_summaries( 

270 log=log, 

271 settings=settings, 

272 dbConn=dbConn 

273 ) 

274 updater._add_distances() 

275 ``` 

276 

277 """ 

278 self.log.debug('starting the ``_add_distances`` method') 

279 

280 extra = "" 

281 if self.transientBucketId: 

282 thisId = self.transientBucketId 

283 extra = "and transientBucketId = %(thisId)s" % locals() 

284 

285 # SELECT THE TRANSIENTS NEEDING UPDATED 

286 sqlQuery = u""" 

287 select best_redshift, transientBucketId from transientBucketSummaries where best_redshift is not null and distanceMpc is null and best_redshift > 0.001 %(extra)s 

288 """ % locals() 

289 rows = readquery( 

290 log=self.log, 

291 sqlQuery=sqlQuery, 

292 dbConn=self.dbConn, 

293 quiet=False 

294 ) 

295 

296 # CONVERT REDSHIFT TO DISTANCE 

297 c = converter(log=self.log) 

298 dictList = [] 

299 for r in rows: 

300 dists = c.redshift_to_distance( 

301 z=r["best_redshift"], 

302 WM=0.3, 

303 WV=0.7, 

304 H0=70.0 

305 ) 

306 dmod = dists["dmod"] 

307 dl_mpc = dists["dl_mpc"] 

308 da_scale = dists["da_scale"] 

309 da_mpc = dists["da_mpc"] 

310 dcmr_mpc = dists["dcmr_mpc"] 

311 dictList.append({ 

312 "transientBucketId": r["transientBucketId"], 

313 "distanceMpc": dl_mpc 

314 }) 

315 

316 if not len(dictList): 

317 return 

318 

319 # USE dbSettings TO ACTIVATE MULTIPROCESSING - INSERT LIST OF 

320 # DICTIONARIES INTO DATABASE 

321 insert_list_of_dictionaries_into_database_tables( 

322 dbConn=self.dbConn, 

323 log=self.log, 

324 dictList=dictList, 

325 dbTableName="transientBucketSummaries", 

326 dateModified=False, 

327 dateCreated=False, 

328 batchSize=2500, 

329 replace=True, 

330 dbSettings=self.settings["database settings"] 

331 ) 

332 

333 self.log.debug('completed the ``_add_distances`` method') 

334 return None 

335 

336 def _update_htm_columns( 

337 self): 

338 """*update the htm columns in the transientSummaries table so we can crossmatch if needed* 

339 """ 

340 self.log.debug('starting the ``_update_htm_columns`` method') 

341 

342 add_htm_ids_to_mysql_database_table( 

343 raColName="raDeg", 

344 declColName="decDeg", 

345 tableName="transientBucketSummaries", 

346 dbConn=self.dbConn, 

347 log=self.log, 

348 primaryIdColumnName="transientBucketId", 

349 dbSettings=self.settings["database settings"] 

350 ) 

351 

352 self.log.debug('completed the ``_update_htm_columns`` method') 

353 return None 

354 

355 # use the tab-trigger below for new method 

356 # xt-class-method