Coverage for sherlock/transient_classifier.py: 70%
832 statements
« prev ^ index » next coverage.py v7.2.2, created at 2023-10-10 13:58 +0000
« prev ^ index » next coverage.py v7.2.2, created at 2023-10-10 13:58 +0000
1#!/usr/local/bin/python
2# encoding: utf-8
3"""
4*classify a set of transients defined by a database query in the sherlock settings file*
6:Author:
7 David Young
8"""
9from __future__ import print_function
10from __future__ import division
11from astrocalc.coords import unit_conversion
12import copy
13from fundamentals.mysql import insert_list_of_dictionaries_into_database_tables
14from fundamentals import fmultiprocess
15import psutil
16from sherlock.commonutils import get_crossmatch_catalogues_column_map
17from sherlock.imports import ned
18from HMpTy.htm import sets
19from HMpTy.mysql import conesearch
20from fundamentals.renderer import list_of_dictionaries
21from fundamentals.mysql import readquery, directory_script_runner, writequery
22from fundamentals import tools
23import numpy as np
24from operator import itemgetter
25from datetime import datetime, date, time, timedelta
27from builtins import zip
28from builtins import str
29from builtins import range
30from builtins import object
31from past.utils import old_div
32import sys
33import os
34import collections
35import codecs
36import re
37import math
38import time
39import inspect
40import yaml
41from random import randint
42os.environ['TERM'] = 'vt100'
44theseBatches = []
45crossmatchArray = []
48class transient_classifier(object):
49 """
50 *The Sherlock Transient Classifier*
52 **Key Arguments**
54 - ``log`` -- logger
55 - ``settings`` -- the settings dictionary
56 - ``update`` -- update the transient database with crossmatch results (boolean)
57 - ``ra`` -- right ascension of a single transient source. Default *False*
58 - ``dec`` -- declination of a single transient source. Default *False*
59 - ``name`` -- the ID of a single transient source. Default *False*
60 - ``verbose`` -- amount of details to print about crossmatches to stdout. 0|1|2 Default *0*
61 - ``updateNed`` -- update the local NED database before running the classifier. Classification will not be as accuracte the NED database is not up-to-date. Default *True*.
62 - ``daemonMode`` -- run sherlock in daemon mode. In daemon mode sherlock remains live and classifies sources as they come into the database. Default *True*
63 - ``updatePeakMags`` -- update peak magnitudes in human-readable annotation of objects (can take some time - best to run occationally)
64 - ``lite`` -- return only a lite version of the results with the topped ranked matches only. Default *False*
65 - ``oneRun`` -- only process one batch of transients, usful for unit testing. Default *False*
67 **Usage**
69 To setup your logger, settings and database connections, please use the ``fundamentals`` package (`see tutorial here <http://fundamentals.readthedocs.io/en/latest/#tutorial>`_).
71 To initiate a transient_classifier object, use the following:
73 .. todo::
75 - update the package tutorial if needed
77 The sherlock classifier can be run in one of two ways. The first is to pass into the coordinates of an object you wish to classify:
79 ```python
80 from sherlock import transient_classifier
81 classifier = transient_classifier(
82 log=log,
83 settings=settings,
84 ra="08:57:57.19",
85 dec="+43:25:44.1",
86 name="PS17gx",
87 verbose=0
88 )
89 classifications, crossmatches = classifier.classify()
90 ```
92 The crossmatches returned are a list of dictionaries giving details of the crossmatched sources. The classifications returned are a list of classifications resulting from these crossmatches. The lists are ordered from most to least likely classification and the indicies for the crossmatch and the classification lists are synced.
94 The second way to run the classifier is to not pass in a coordinate set and therefore cause sherlock to run the classifier on the transient database referenced in the sherlock settings file:
96 ```python
97 from sherlock import transient_classifier
98 classifier = transient_classifier(
99 log=log,
100 settings=settings,
101 update=True
102 )
103 classifier.classify()
104 ```
106 Here the transient list is selected out of the database using the ``transient query`` value in the settings file:
108 ```yaml
109 database settings:
110 transients:
111 user: myusername
112 password: mypassword
113 db: nice_transients
114 host: 127.0.0.1
115 transient table: transientBucket
116 transient query: "select primaryKeyId as 'id', transientBucketId as 'alt_id', raDeg 'ra', decDeg 'dec', name 'name', sherlockClassification as 'object_classification'
117 from transientBucket where object_classification is null
118 transient primary id column: primaryKeyId
119 transient classification column: sherlockClassification
120 tunnel: False
121 ```
123 By setting ``update=True`` the classifier will update the ``sherlockClassification`` column of the ``transient table`` with new classification and populate the ``sherlock_crossmatches`` table with key details of the crossmatched sources from the catalogues database. By setting ``update=False`` results are printed to stdout but the database is not updated (useful for dry runs and testing new algorithms),
126 .. todo ::
128 - update key arguments values and definitions with defaults
129 - update return values and definitions
130 - update usage examples and text
131 - update docstring text
132 - check sublime snippet exists
133 - clip any useful text to docs mindmap
134 - regenerate the docs and check redendering of this docstring
135 """
136 # INITIALISATION
138 def __init__(
139 self,
140 log,
141 settings=False,
142 update=False,
143 ra=False,
144 dec=False,
145 name=False,
146 verbose=0,
147 updateNed=True,
148 daemonMode=False,
149 updatePeakMags=True,
150 oneRun=False,
151 lite=False
152 ):
153 self.log = log
154 log.debug("instansiating a new 'classifier' object")
155 self.settings = settings
156 self.update = update
157 self.ra = ra
158 self.dec = dec
159 self.name = name
160 self.cl = False
161 self.verbose = verbose
162 self.updateNed = updateNed
163 self.daemonMode = daemonMode
164 self.updatePeakMags = updatePeakMags
165 self.oneRun = oneRun
166 self.lite = lite
167 self.filterPreference = [
168 "R", "_r", "G", "V", "_g", "B", "I", "_i", "_z", "J", "H", "K", "U", "_u", "_y", "W1", "unkMag"
169 ]
171 # COLLECT ADVANCED SETTINGS IF AVAILABLE
172 parentDirectory = os.path.dirname(__file__)
173 advs = parentDirectory + "/advanced_settings.yaml"
174 level = 0
175 exists = False
176 count = 1
177 while not exists and len(advs) and count < 10:
178 count += 1
179 level -= 1
180 exists = os.path.exists(advs)
181 if not exists:
182 advs = "/".join(parentDirectory.split("/")
183 [:level]) + "/advanced_settings.yaml"
184 print(advs)
185 if not exists:
186 advs = {}
187 else:
188 with open(advs, 'r') as stream:
189 advs = yaml.safe_load(stream)
190 # MERGE ADVANCED SETTINGS AND USER SETTINGS (USER SETTINGS OVERRIDE)
191 self.settings = {**advs, **self.settings}
193 # INITIAL ACTIONS
194 # SETUP DATABASE CONNECTIONS
195 # SETUP ALL DATABASE CONNECTIONS
196 from sherlock import database
197 db = database(
198 log=self.log,
199 settings=self.settings
200 )
201 dbConns, dbVersions = db.connect()
202 self.dbVersions = dbVersions
203 self.transientsDbConn = dbConns["transients"]
204 self.cataloguesDbConn = dbConns["catalogues"]
206 # SIZE OF BATCHES TO SPLIT TRANSIENT INTO BEFORE CLASSIFYING
207 self.largeBatchSize = self.settings["database-batch-size"]
208 self.miniBatchSize = 1000
210 # LITE VERSION CANNOT BE RUN ON A DATABASE QUERY AS YET
211 if self.ra == False:
212 self.lite = False
214 # IS SHERLOCK CLASSIFIER BEING QUERIED FROM THE COMMAND-LINE?
215 if self.ra and self.dec:
216 self.cl = True
217 if not self.name:
218 self.name = "Transient"
220 # ASTROCALC UNIT CONVERTER OBJECT
221 self.converter = unit_conversion(
222 log=self.log
223 )
225 if self.ra and not isinstance(self.ra, float) and ":" in self.ra:
226 # ASTROCALC UNIT CONVERTER OBJECT
227 self.ra = self.converter.ra_sexegesimal_to_decimal(
228 ra=self.ra
229 )
230 self.dec = self.converter.dec_sexegesimal_to_decimal(
231 dec=self.dec
232 )
234 # DATETIME REGEX - EXPENSIVE OPERATION, LET"S JUST DO IT ONCE
235 self.reDatetime = re.compile('^[0-9]{4}-[0-9]{2}-[0-9]{2}T')
237 return None
239 def classify(self):
240 """
241 *classify the transients selected from the transient selection query in the settings file or passed in via the CL or other code*
243 **Return**
245 - ``crossmatches`` -- list of dictionaries of crossmatched associated sources
246 - ``classifications`` -- the classifications assigned to the transients post-crossmatches (dictionary of rank ordered list of classifications)
249 See class docstring for usage.
251 .. todo ::
253 - update key arguments values and definitions with defaults
254 - update return values and definitions
255 - update usage examples and text
256 - update docstring text
257 - check sublime snippet exists
258 - clip any useful text to docs mindmap
259 - regenerate the docs and check redendering of this docstring
260 """
262 global theseBatches
263 global crossmatchArray
265 self.log.debug('starting the ``classify`` method')
267 remaining = 1
269 # THE COLUMN MAPS - WHICH COLUMNS IN THE CATALOGUE TABLES = RA, DEC,
270 # REDSHIFT, MAG ETC
271 colMaps = get_crossmatch_catalogues_column_map(
272 log=self.log,
273 dbConn=self.cataloguesDbConn
274 )
276 if self.transientsDbConn and self.update:
277 self._create_tables_if_not_exist()
279 import time
280 start_time = time.time()
282 # COUNT SEARCHES
283 sa = self.settings["search algorithm"]
284 searchCount = 0
285 brightnessFilters = ["bright", "faint", "general"]
286 for search_name, searchPara in list(sa.items()):
287 for bf in brightnessFilters:
288 if bf in searchPara:
289 searchCount += 1
291 cpuCount = psutil.cpu_count()
292 if searchCount > cpuCount:
293 searchCount = cpuCount
295 miniBatchSize = self.miniBatchSize
297 while remaining:
299 # IF A TRANSIENT HAS NOT BEEN PASSED IN VIA THE COMMAND-LINE, THEN
300 # QUERY THE TRANSIENT DATABASE
301 if not self.ra and not self.dec:
303 # COUNT REMAINING TRANSIENTS
304 from fundamentals.mysql import readquery
305 sqlQuery = self.settings["database settings"][
306 "transients"]["transient count"]
307 thisInt = randint(0, 100)
308 if "where" in sqlQuery:
309 sqlQuery = sqlQuery.replace(
310 "where", "where %(thisInt)s=%(thisInt)s and " % locals())
312 if remaining == 1 or remaining < self.largeBatchSize:
313 rows = readquery(
314 log=self.log,
315 sqlQuery=sqlQuery,
316 dbConn=self.transientsDbConn,
317 )
318 remaining = rows[0]["count(*)"]
319 else:
320 remaining = remaining - self.largeBatchSize
322 print(
323 "%(remaining)s transient sources requiring a classification remain" % locals())
325 # START THE TIME TO TRACK CLASSIFICATION SPPED
326 start_time = time.time()
328 # A LIST OF DICTIONARIES OF TRANSIENT METADATA
329 transientsMetadataList = self._get_transient_metadata_from_database_list()
331 count = len(transientsMetadataList)
332 print(
333 " now classifying the next %(count)s transient sources" % locals())
335 # EXAMPLE OF TRANSIENT METADATA
336 # { 'name': 'PS17gx',
337 # 'alt_id': 'PS17gx',
338 # 'object_classification': 'SN',
339 # 'dec': '+43:25:44.1',
340 # 'id': 1,
341 # 'ra': '08:57:57.19'}
342 # TRANSIENT PASSED VIA COMMAND-LINE
343 else:
344 # CONVERT SINGLE TRANSIENTS TO LIST
345 if not isinstance(self.ra, list):
346 self.ra = [self.ra]
347 self.dec = [self.dec]
348 self.name = [self.name]
350 # GIVEN TRANSIENTS UNIQUE NAMES IF NOT PROVIDED
351 if not self.name[0]:
352 self.name = []
353 for i, v in enumerate(self.ra):
354 self.name.append("transient_%(i)05d" % locals())
356 transientsMetadataList = []
357 for r, d, n in zip(self.ra, self.dec, self.name):
358 transient = {
359 'name': n,
360 'object_classification': None,
361 'dec': d,
362 'id': n,
363 'ra': r
364 }
365 transientsMetadataList.append(transient)
366 remaining = 0
368 if self.oneRun:
369 remaining = 0
371 if len(transientsMetadataList) == 0:
372 if self.daemonMode == False:
373 remaining = 0
374 print("No transients need classified")
375 return None, None
376 else:
377 print(
378 "No remaining transients need classified, will try again in 5 mins")
379 time.sleep("10")
381 # FROM THE LOCATIONS OF THE TRANSIENTS, CHECK IF OUR LOCAL NED DATABASE
382 # NEEDS UPDATED
383 if self.updateNed:
385 self._update_ned_stream(
386 transientsMetadataList=transientsMetadataList
387 )
389 # SOME TESTING SHOWED THAT 25 IS GOOD
390 total = len(transientsMetadataList)
391 batches = int((old_div(float(total), float(miniBatchSize))) + 1.)
393 if batches == 0:
394 batches = 1
396 start = 0
397 end = 0
398 theseBatches = []
399 for i in range(batches):
400 end = end + miniBatchSize
401 start = i * miniBatchSize
402 thisBatch = transientsMetadataList[start:end]
403 theseBatches.append(thisBatch)
405 if self.verbose > 1:
406 print("BATCH SIZE = %(total)s" % locals())
407 print("MINI BATCH SIZE = %(batches)s x %(miniBatchSize)s" % locals())
409 poolSize = self.settings["cpu-pool-size"]
410 if poolSize and batches < poolSize:
411 poolSize = batches
413 start_time2 = time.time()
415 if self.verbose > 1:
416 print("START CROSSMATCH")
418 crossmatchArray = fmultiprocess(log=self.log, function=_crossmatch_transients_against_catalogues,
419 inputArray=list(range(len(theseBatches))), poolSize=poolSize, settings=self.settings, colMaps=colMaps)
421 if self.verbose > 1:
422 print("FINISH CROSSMATCH/START RANKING: %d" %
423 (time.time() - start_time2,))
424 start_time2 = time.time()
426 classifications = {}
427 crossmatches = []
429 for sublist in crossmatchArray:
430 sublist = sorted(
431 sublist, key=itemgetter('transient_object_id'))
433 # REORGANISE INTO INDIVIDUAL TRANSIENTS FOR RANKING AND
434 # TOP-LEVEL CLASSIFICATION EXTRACTION
436 batch = []
437 if len(sublist) != 0:
438 transientId = sublist[0]['transient_object_id']
439 for s in sublist:
440 if s['transient_object_id'] != transientId:
441 # RANK TRANSIENT CROSSMATCH BATCH
442 cl, cr = self._rank_classifications(
443 batch, colMaps)
444 crossmatches.extend(cr)
445 classifications = dict(
446 list(classifications.items()) + list(cl.items()))
448 transientId = s['transient_object_id']
449 batch = [s]
450 else:
451 batch.append(s)
453 # RANK FINAL BATCH
454 cl, cr = self._rank_classifications(
455 batch, colMaps)
456 classifications = dict(
457 list(classifications.items()) + list(cl.items()))
458 crossmatches.extend(cr)
460 for t in transientsMetadataList:
461 if t["id"] not in classifications:
462 classifications[t["id"]] = ["ORPHAN"]
464 # UPDATE THE TRANSIENT DATABASE IF UPDATE REQUESTED (ADD DATA TO
465 # tcs_crossmatch_table AND A CLASSIFICATION TO THE ORIGINAL TRANSIENT
466 # TABLE)
467 if self.verbose > 1:
468 print("FINISH RANKING/START UPDATING TRANSIENT DB: %d" %
469 (time.time() - start_time2,))
470 start_time2 = time.time()
471 if self.update and not self.ra:
472 self._update_transient_database(
473 crossmatches=crossmatches,
474 classifications=classifications,
475 transientsMetadataList=transientsMetadataList,
476 colMaps=colMaps
477 )
479 if self.verbose > 1:
480 print("FINISH UPDATING TRANSIENT DB/START ANNOTATING TRANSIENT DB: %d" %
481 (time.time() - start_time2,))
482 start_time2 = time.time()
484 # COMMAND-LINE SINGLE CLASSIFICATION
485 if self.ra:
486 classifications = self.update_classification_annotations_and_summaries(
487 False, True, crossmatches, classifications)
488 for k, v in classifications.items():
489 if len(v) == 1 and v[0] == "ORPHAN":
490 v.append(
491 "No contexual information is available for this transient")
493 if self.lite != False:
494 crossmatches = self._lighten_return(crossmatches)
496 if self.cl:
497 self._print_results_to_stdout(
498 classifications=classifications,
499 crossmatches=crossmatches
500 )
502 return classifications, crossmatches
504 if self.updatePeakMags and self.settings["database settings"]["transients"]["transient peak magnitude query"]:
505 self.update_peak_magnitudes()
507 # BULK RUN -- NOT A COMMAND-LINE SINGLE CLASSIFICATION
508 self.update_classification_annotations_and_summaries(
509 self.updatePeakMags)
511 print("FINISH ANNOTATING TRANSIENT DB: %d" %
512 (time.time() - start_time2,))
513 start_time2 = time.time()
515 classificationRate = old_div(count, (time.time() - start_time))
516 print(
517 "Sherlock is classify at a rate of %(classificationRate)2.1f transients/sec" % locals())
519 self.log.debug('completed the ``classify`` method')
520 return None, None
522 def _get_transient_metadata_from_database_list(
523 self):
524 """use the transient query in the settings file to generate a list of transients to corssmatch and classify
526 **Return**
529 - ``transientsMetadataList``
531 .. todo ::
533 - update key arguments values and definitions with defaults
534 - update return values and definitions
535 - update usage examples and text
536 - update docstring text
537 - check sublime snippet exists
538 - clip any useful text to docs mindmap
539 - regenerate the docs and check redendering of this docstring
540 """
541 self.log.debug(
542 'starting the ``_get_transient_metadata_from_database_list`` method')
544 sqlQuery = self.settings["database settings"][
545 "transients"]["transient query"] + " limit " + str(self.largeBatchSize)
547 thisInt = randint(0, 100)
548 if "where" in sqlQuery:
549 sqlQuery = sqlQuery.replace(
550 "where", "where %(thisInt)s=%(thisInt)s and " % locals())
552 transientsMetadataList = readquery(
553 log=self.log,
554 sqlQuery=sqlQuery,
555 dbConn=self.transientsDbConn,
556 quiet=False
557 )
559 self.log.debug(
560 'completed the ``_get_transient_metadata_from_database_list`` method')
561 return transientsMetadataList
563 def _update_ned_stream(
564 self,
565 transientsMetadataList
566 ):
567 """ update the NED stream within the catalogues database at the locations of the transients
569 **Key Arguments**
571 - ``transientsMetadataList`` -- the list of transient metadata lifted from the database.
574 .. todo ::
576 - update key arguments values and definitions with defaults
577 - update return values and definitions
578 - update usage examples and text
579 - update docstring text
580 - check sublime snippet exists
581 - clip any useful text to docs mindmap
582 - regenerate the docs and check redendering of this docstring
583 """
584 self.log.debug('starting the ``_update_ned_stream`` method')
586 coordinateList = []
587 for i in transientsMetadataList:
588 # thisList = str(i["ra"]) + " " + str(i["dec"])
589 thisList = (i["ra"], i["dec"])
590 coordinateList.append(thisList)
592 coordinateList = self._remove_previous_ned_queries(
593 coordinateList=coordinateList
594 )
596 # MINIMISE COORDINATES IN LIST TO REDUCE NUMBER OF REQUIRE NED QUERIES
597 coordinateList = self._consolidate_coordinateList(
598 coordinateList=coordinateList
599 )
601 stream = ned(
602 log=self.log,
603 settings=self.settings,
604 coordinateList=coordinateList,
605 radiusArcsec=self.settings["ned stream search radius arcec"]
606 )
607 stream.ingest()
609 sqlQuery = """SET session sql_mode = "";""" % locals(
610 )
611 writequery(
612 log=self.log,
613 sqlQuery=sqlQuery,
614 dbConn=self.cataloguesDbConn
615 )
616 sqlQuery = """update tcs_cat_ned_stream set magnitude = CAST(`magnitude_filter` AS DECIMAL(5,2)) where magnitude is null and magnitude_filter is not null;""" % locals(
617 )
618 writequery(
619 log=self.log,
620 sqlQuery=sqlQuery,
621 dbConn=self.cataloguesDbConn
622 )
624 self.log.debug('completed the ``_update_ned_stream`` method')
625 return None
627 def _remove_previous_ned_queries(
628 self,
629 coordinateList):
630 """iterate through the transient locations to see if we have recent local NED coverage of that area already
632 **Key Arguments**
634 - ``coordinateList`` -- set of coordinate to check for previous queries
637 **Return**
639 - ``updatedCoordinateList`` -- coordinate list with previous queries removed
642 .. todo ::
644 - update key arguments values and definitions with defaults
645 - update return values and definitions
646 - update usage examples and text
647 - update docstring text
648 - check sublime snippet exists
649 - clip any useful text to docs mindmap
650 - regenerate the docs and check redendering of this docstring
651 """
652 self.log.debug('starting the ``_remove_previous_ned_queries`` method')
654 # 1 DEGREE QUERY RADIUS
655 radius = 60. * 60.
656 updatedCoordinateList = []
657 keepers = []
659 # CALCULATE THE OLDEST RESULTS LIMIT
660 now = datetime.now()
661 td = timedelta(
662 days=self.settings["ned stream refresh rate in days"])
663 refreshLimit = now - td
664 refreshLimit = refreshLimit.strftime("%Y-%m-%d %H:%M:%S")
666 raList = []
667 raList[:] = [c[0] for c in coordinateList]
668 decList = []
669 decList[:] = [c[1] for c in coordinateList]
671 # MATCH COORDINATES AGAINST PREVIOUS NED SEARCHES
672 cs = conesearch(
673 log=self.log,
674 dbConn=self.cataloguesDbConn,
675 tableName="tcs_helper_ned_query_history",
676 columns="*",
677 ra=raList,
678 dec=decList,
679 radiusArcsec=radius,
680 separations=True,
681 distinct=True,
682 sqlWhere="dateQueried > '%(refreshLimit)s'" % locals(),
683 closest=False
684 )
685 matchIndies, matches = cs.search()
687 # DETERMINE WHICH COORDINATES REQUIRE A NED QUERY
688 curatedMatchIndices = []
689 curatedMatches = []
690 for i, m in zip(matchIndies, matches.list):
691 match = False
692 row = m
693 row["separationArcsec"] = row["cmSepArcsec"]
694 raStream = row["raDeg"]
695 decStream = row["decDeg"]
696 radiusStream = row["arcsecRadius"]
697 dateStream = row["dateQueried"]
698 angularSeparation = row["separationArcsec"]
700 if angularSeparation + self.settings["first pass ned search radius arcec"] < radiusStream:
701 curatedMatchIndices.append(i)
702 curatedMatches.append(m)
704 # NON MATCHES
705 for i, v in enumerate(coordinateList):
706 if i not in curatedMatchIndices:
707 updatedCoordinateList.append(v)
709 self.log.debug('completed the ``_remove_previous_ned_queries`` method')
710 return updatedCoordinateList
712 def _update_transient_database(
713 self,
714 crossmatches,
715 classifications,
716 transientsMetadataList,
717 colMaps):
718 """ update transient database with classifications and crossmatch results
720 **Key Arguments**
722 - ``crossmatches`` -- the crossmatches and associations resulting from the catlaogue crossmatches
723 - ``classifications`` -- the classifications assigned to the transients post-crossmatches (dictionary of rank ordered list of classifications)
724 - ``transientsMetadataList`` -- the list of transient metadata lifted from the database.
725 - ``colMaps`` -- maps of the important column names for each table/view in the crossmatch-catalogues database
728 .. todo ::
730 - update key arguments values and definitions with defaults
731 - update return values and definitions
732 - update usage examples and text
733 - update docstring text
734 - check sublime snippet exists
735 - clip any useful text to docs mindmap
736 - regenerate the docs and check redendering of this docstring
737 """
739 self.log.debug('starting the ``_update_transient_database`` method')
741 import time
742 start_time = time.time()
743 print("UPDATING TRANSIENTS DATABASE WITH RESULTS")
744 print("DELETING OLD RESULTS")
746 now = datetime.now()
747 now = now.strftime("%Y-%m-%d_%H-%M-%S-%f")
749 transientTable = self.settings["database settings"][
750 "transients"]["transient table"]
751 transientTableClassCol = self.settings["database settings"][
752 "transients"]["transient classification column"]
753 transientTableIdCol = self.settings["database settings"][
754 "transients"]["transient primary id column"]
756 # COMBINE ALL CROSSMATCHES INTO A LIST OF DICTIONARIES TO DUMP INTO
757 # DATABASE TABLE
758 transientIDs = [str(c)
759 for c in list(classifications.keys())]
760 transientIDs = ",".join(transientIDs)
762 # REMOVE PREVIOUS MATCHES
763 sqlQuery = """delete from sherlock_crossmatches where transient_object_id in (%(transientIDs)s);""" % locals(
764 )
765 writequery(
766 log=self.log,
767 sqlQuery=sqlQuery,
768 dbConn=self.transientsDbConn,
769 )
770 sqlQuery = """delete from sherlock_classifications where transient_object_id in (%(transientIDs)s);""" % locals(
771 )
772 writequery(
773 log=self.log,
774 sqlQuery=sqlQuery,
775 dbConn=self.transientsDbConn,
776 )
778 print("FINISHED DELETING OLD RESULTS/ADDING TO CROSSMATCHES: %d" %
779 (time.time() - start_time,))
780 start_time = time.time()
782 if len(crossmatches):
783 insert_list_of_dictionaries_into_database_tables(
784 dbConn=self.transientsDbConn,
785 log=self.log,
786 dictList=crossmatches,
787 dbTableName="sherlock_crossmatches",
788 dateModified=True,
789 batchSize=10000,
790 replace=True,
791 dbSettings=self.settings["database settings"][
792 "transients"]
793 )
795 print("FINISHED ADDING TO CROSSMATCHES/UPDATING CLASSIFICATIONS IN TRANSIENT TABLE: %d" %
796 (time.time() - start_time,))
797 start_time = time.time()
799 sqlQuery = ""
800 inserts = []
801 for k, v in list(classifications.items()):
802 thisInsert = {
803 "transient_object_id": k,
804 "classification": v[0]
805 }
806 inserts.append(thisInsert)
808 print("FINISHED UPDATING CLASSIFICATIONS IN TRANSIENT TABLE/UPDATING sherlock_classifications TABLE: %d" %
809 (time.time() - start_time,))
810 start_time = time.time()
812 insert_list_of_dictionaries_into_database_tables(
813 dbConn=self.transientsDbConn,
814 log=self.log,
815 dictList=inserts,
816 dbTableName="sherlock_classifications",
817 dateModified=True,
818 batchSize=10000,
819 replace=True,
820 dbSettings=self.settings["database settings"][
821 "transients"]
822 )
824 print("FINISHED UPDATING sherlock_classifications TABLE: %d" %
825 (time.time() - start_time,))
826 start_time = time.time()
828 self.log.debug('completed the ``_update_transient_database`` method')
829 return None
831 def _rank_classifications(
832 self,
833 crossmatchArray,
834 colMaps):
835 """*rank the classifications returned from the catalogue crossmatcher, annotate the results with a classification rank-number (most likely = 1) and a rank-score (weight of classification)*
837 **Key Arguments**
839 - ``crossmatchArrayIndex`` -- the index of list of unranked crossmatch classifications
840 - ``colMaps`` -- dictionary of dictionaries with the name of the database-view (e.g. `tcs_view_agn_milliquas_v4_5`) as the key and the column-name dictary map as value (`{view_name: {columnMap}}`).
843 **Return**
845 - ``classifications`` -- the classifications assigned to the transients post-crossmatches
846 - ``crossmatches`` -- the crossmatches annotated with rankings and rank-scores
849 .. todo ::
851 - update key arguments values and definitions with defaults
852 - update return values and definitions
853 - update usage examples and text
854 - update docstring text
855 - check sublime snippet exists
856 - clip any useful text to docs mindmap
857 - regenerate the docs and check redendering of this docstring
858 """
859 self.log.debug('starting the ``_rank_classifications`` method')
861 crossmatches = crossmatchArray
863 # GROUP CROSSMATCHES INTO DISTINCT SOURCES (DUPLICATE ENTRIES OF THE
864 # SAME ASTROPHYSICAL SOURCE ACROSS MULTIPLE CATALOGUES)
865 ra, dec = list(zip(*[(r["raDeg"], r["decDeg"]) for r in crossmatches]))
867 from HMpTy.htm import sets
868 xmatcher = sets(
869 log=self.log,
870 ra=ra,
871 dec=dec,
872 radius=1. / (60. * 60.), # in degrees
873 sourceList=crossmatches
874 )
875 groupedMatches = xmatcher.match
877 associatationTypeOrder = ["AGN", "CV", "NT", "SN", "VS", "BS"]
879 # ADD DISTINCT-SOURCE KEY
880 dupKey = 0
881 distinctMatches = []
882 for x in groupedMatches:
883 dupKey += 1
884 mergedMatch = copy.deepcopy(x[0])
885 mergedMatch["merged_rank"] = int(dupKey)
887 # ADD OTHER ESSENTIAL KEYS
888 for e in ['z', 'photoZ', 'photoZErr']:
889 if e not in mergedMatch:
890 mergedMatch[e] = None
891 bestQualityCatalogue = colMaps[mergedMatch[
892 "catalogue_view_name"]]["object_type_accuracy"]
893 bestDirectDistance = {
894 "direct_distance": mergedMatch["direct_distance"],
895 "direct_distance_modulus": mergedMatch["direct_distance_modulus"],
896 "direct_distance_scale": mergedMatch["direct_distance_scale"],
897 "qual": colMaps[mergedMatch["catalogue_view_name"]]["object_type_accuracy"]
898 }
899 if not mergedMatch["direct_distance"]:
900 bestDirectDistance["qual"] = 0
902 bestSpecz = {
903 "z": mergedMatch["z"],
904 "distance": mergedMatch["distance"],
905 "distance_modulus": mergedMatch["distance_modulus"],
906 "scale": mergedMatch["scale"],
907 "qual": colMaps[mergedMatch["catalogue_view_name"]]["object_type_accuracy"]
908 }
909 if not mergedMatch["distance"]:
910 bestSpecz["qual"] = 0
912 bestPhotoz = {
913 "photoZ": mergedMatch["photoZ"],
914 "photoZErr": mergedMatch["photoZErr"],
915 "qual": colMaps[mergedMatch["catalogue_view_name"]]["object_type_accuracy"]
916 }
917 if not mergedMatch["photoZ"]:
918 bestPhotoz["qual"] = 0
920 # ORDER THESE FIRST IN NAME LISTING
921 mergedMatch["search_name"] = None
922 mergedMatch["catalogue_object_id"] = None
923 primeCats = ["NED", "SDSS", "MILLIQUAS"]
924 for cat in primeCats:
925 for i, m in enumerate(x):
926 # MERGE SEARCH NAMES
927 snippet = m["search_name"].split(" ")[0].upper()
928 if cat.upper() in snippet:
929 if not mergedMatch["search_name"]:
930 mergedMatch["search_name"] = m["search_name"].split(" ")[
931 0].upper()
932 elif "/" not in mergedMatch["search_name"] and snippet not in mergedMatch["search_name"].upper():
933 mergedMatch["search_name"] = mergedMatch["search_name"].split(
934 " ")[0].upper() + "/" + m["search_name"].split(" ")[0].upper()
935 elif snippet not in mergedMatch["search_name"].upper():
936 mergedMatch[
937 "search_name"] += "/" + m["search_name"].split(" ")[0].upper()
938 elif "/" not in mergedMatch["search_name"]:
939 mergedMatch["search_name"] = mergedMatch["search_name"].split(
940 " ")[0].upper()
941 mergedMatch["catalogue_table_name"] = mergedMatch[
942 "search_name"]
944 # MERGE CATALOGUE SOURCE NAMES
945 if not mergedMatch["catalogue_object_id"]:
946 mergedMatch["catalogue_object_id"] = str(
947 m["catalogue_object_id"])
949 # NOW ADD THE REST
950 for i, m in enumerate(x):
951 # MERGE SEARCH NAMES
952 snippet = m["search_name"].split(" ")[0].upper()
953 if snippet not in primeCats:
954 if not mergedMatch["search_name"]:
955 mergedMatch["search_name"] = m["search_name"].split(" ")[
956 0].upper()
957 elif "/" not in mergedMatch["search_name"] and snippet not in mergedMatch["search_name"].upper():
958 mergedMatch["search_name"] = mergedMatch["search_name"].split(
959 " ")[0].upper() + "/" + m["search_name"].split(" ")[0].upper()
960 elif snippet not in mergedMatch["search_name"].upper():
961 mergedMatch[
962 "search_name"] += "/" + m["search_name"].split(" ")[0].upper()
963 elif "/" not in mergedMatch["search_name"]:
964 mergedMatch["search_name"] = mergedMatch["search_name"].split(
965 " ")[0].upper()
966 mergedMatch["catalogue_table_name"] = mergedMatch[
967 "search_name"]
969 # MERGE CATALOGUE SOURCE NAMES
970 if not mergedMatch["catalogue_object_id"]:
971 mergedMatch["catalogue_object_id"] = str(
972 m["catalogue_object_id"])
973 # else:
974 # mergedMatch["catalogue_object_id"] = str(
975 # mergedMatch["catalogue_object_id"])
976 # m["catalogue_object_id"] = str(
977 # m["catalogue_object_id"])
978 # if m["catalogue_object_id"].replace(" ", "").lower() not in mergedMatch["catalogue_object_id"].replace(" ", "").lower():
979 # mergedMatch["catalogue_object_id"] += "/" + \
980 # m["catalogue_object_id"]
982 for i, m in enumerate(x):
983 m["merged_rank"] = int(dupKey)
984 if i > 0:
985 # MERGE ALL BEST MAGNITUDE MEASUREMENTS
986 for f in self.filterPreference:
988 if f in m and m[f] and (f not in mergedMatch or (f + "Err" in mergedMatch and f + "Err" in m and (mergedMatch[f + "Err"] == None or (m[f + "Err"] and mergedMatch[f + "Err"] > m[f + "Err"])))):
989 mergedMatch[f] = m[f]
990 try:
991 mergedMatch[f + "Err"] = m[f + "Err"]
992 except:
993 pass
994 mergedMatch["original_search_radius_arcsec"] = "multiple"
995 mergedMatch["catalogue_object_subtype"] = "multiple"
996 mergedMatch["catalogue_view_name"] = "multiple"
998 # DETERMINE BEST CLASSIFICATION
999 if mergedMatch["classificationReliability"] == 3 and m["classificationReliability"] < 3:
1000 mergedMatch["association_type"] = m["association_type"]
1001 mergedMatch["catalogue_object_type"] = m[
1002 "catalogue_object_type"]
1003 mergedMatch["classificationReliability"] = m[
1004 "classificationReliability"]
1006 if m["classificationReliability"] != 3 and colMaps[m["catalogue_view_name"]]["object_type_accuracy"] > bestQualityCatalogue:
1007 bestQualityCatalogue = colMaps[
1008 m["catalogue_view_name"]]["object_type_accuracy"]
1009 mergedMatch["association_type"] = m["association_type"]
1010 mergedMatch["catalogue_object_type"] = m[
1011 "catalogue_object_type"]
1012 mergedMatch["classificationReliability"] = m[
1013 "classificationReliability"]
1015 if m["classificationReliability"] != 3 and colMaps[m["catalogue_view_name"]]["object_type_accuracy"] == bestQualityCatalogue and m["association_type"] in associatationTypeOrder and (mergedMatch["association_type"] not in associatationTypeOrder or associatationTypeOrder.index(m["association_type"]) < associatationTypeOrder.index(mergedMatch["association_type"])):
1016 mergedMatch["association_type"] = m["association_type"]
1017 mergedMatch["catalogue_object_type"] = m[
1018 "catalogue_object_type"]
1019 mergedMatch["classificationReliability"] = m[
1020 "classificationReliability"]
1022 # FIND BEST DISTANCES
1023 if "direct_distance" in m and m["direct_distance"] and colMaps[m["catalogue_view_name"]]["object_type_accuracy"] > bestDirectDistance["qual"]:
1024 bestDirectDistance = {
1025 "direct_distance": m["direct_distance"],
1026 "direct_distance_modulus": m["direct_distance_modulus"],
1027 "direct_distance_scale": m["direct_distance_scale"],
1028 "catalogue_object_type": m["catalogue_object_type"],
1029 "qual": colMaps[m["catalogue_view_name"]]["object_type_accuracy"]
1030 }
1031 # FIND BEST SPEC-Z
1032 if "z" in m and m["z"] and colMaps[m["catalogue_view_name"]]["object_type_accuracy"] > bestSpecz["qual"]:
1033 bestSpecz = {
1034 "z": m["z"],
1035 "distance": m["distance"],
1036 "distance_modulus": m["distance_modulus"],
1037 "scale": m["scale"],
1038 "catalogue_object_type": m["catalogue_object_type"],
1039 "qual": colMaps[m["catalogue_view_name"]]["object_type_accuracy"]
1040 }
1041 # FIND BEST PHOT-Z
1042 if "photoZ" in m and m["photoZ"] and colMaps[m["catalogue_view_name"]]["object_type_accuracy"] > bestPhotoz["qual"]:
1043 bestPhotoz = {
1044 "photoZ": m["photoZ"],
1045 "photoZErr": m["photoZErr"],
1046 "distance": m["distance"],
1047 "distance_modulus": m["distance_modulus"],
1048 "scale": m["scale"],
1049 "catalogue_object_type": m["catalogue_object_type"],
1050 "qual": colMaps[m["catalogue_view_name"]]["object_type_accuracy"]
1051 }
1052 # CLOSEST ANGULAR SEP & COORDINATES
1053 if m["separationArcsec"] < mergedMatch["separationArcsec"]:
1054 mergedMatch["separationArcsec"] = m["separationArcsec"]
1055 mergedMatch["raDeg"] = m["raDeg"]
1056 mergedMatch["decDeg"] = m["decDeg"]
1058 # MERGE THE BEST RESULTS
1059 for l in [bestPhotoz, bestSpecz, bestDirectDistance]:
1060 for k, v in list(l.items()):
1061 if k != "qual" and v:
1062 mergedMatch[k] = v
1064 mergedMatch["catalogue_object_id"] = str(mergedMatch[
1065 "catalogue_object_id"]).replace(" ", "")
1067 # RECALULATE PHYSICAL DISTANCE SEPARATION
1068 if mergedMatch["direct_distance_scale"]:
1069 mergedMatch["physical_separation_kpc"] = mergedMatch[
1070 "direct_distance_scale"] * mergedMatch["separationArcsec"]
1072 elif mergedMatch["scale"]:
1073 mergedMatch["physical_separation_kpc"] = mergedMatch[
1074 "scale"] * mergedMatch["separationArcsec"]
1076 if "/" in mergedMatch["search_name"]:
1077 mergedMatch["search_name"] = "multiple"
1079 distinctMatches.append(mergedMatch)
1081 crossmatches = []
1082 for xm, gm in zip(distinctMatches, groupedMatches):
1083 # SPEC-Z GALAXIES
1084 if (xm["physical_separation_kpc"] is not None and xm["physical_separation_kpc"] != "null" and xm["physical_separation_kpc"] < 20. and (("z" in xm and xm["z"] is not None) or "photoZ" not in xm or xm["photoZ"] is None or xm["photoZ"] < 0.)):
1085 rankScore = xm["classificationReliability"] * 1000 + 2. - \
1086 (50 - old_div(xm["physical_separation_kpc"], 20))
1087 # PHOTO-Z GALAXIES
1088 elif (xm["physical_separation_kpc"] is not None and xm["physical_separation_kpc"] != "null" and xm["physical_separation_kpc"] < 20. and xm["association_type"] == "SN"):
1089 rankScore = xm["classificationReliability"] * 1000 + 5 - \
1090 (50 - old_div(xm["physical_separation_kpc"], 20))
1091 # NOT SPEC-Z, NON PHOTO-Z GALAXIES & PHOTO-Z GALAXIES
1092 elif (xm["association_type"] == "SN"):
1093 rankScore = xm["classificationReliability"] * 1000 + 5.
1094 # VS
1095 elif (xm["association_type"] == "VS"):
1096 rankScore = xm["classificationReliability"] * \
1097 1000 + xm["separationArcsec"] + 2.
1098 # BS
1099 elif (xm["association_type"] == "BS"):
1100 rankScore = xm["classificationReliability"] * \
1101 1000 + xm["separationArcsec"]
1102 else:
1103 rankScore = xm["classificationReliability"] * \
1104 1000 + xm["separationArcsec"] + 10.
1105 xm["rankScore"] = rankScore
1106 crossmatches.append(xm)
1107 if len(gm) > 1:
1108 for g in gm:
1109 g["rankScore"] = rankScore
1111 crossmatches = sorted(
1112 crossmatches, key=itemgetter('rankScore'), reverse=False)
1113 crossmatches = sorted(
1114 crossmatches, key=itemgetter('transient_object_id'))
1116 transient_object_id = None
1117 uniqueIndexCheck = []
1118 classifications = {}
1119 crossmatchesKeep = []
1120 rank = 0
1121 transClass = []
1122 for xm in crossmatches:
1123 rank += 1
1124 if rank == 1:
1125 transClass.append(xm["association_type"])
1126 classifications[xm["transient_object_id"]] = transClass
1127 if rank == 1 or self.lite == False:
1128 xm["rank"] = rank
1129 crossmatchesKeep.append(xm)
1130 crossmatches = crossmatchesKeep
1132 crossmatchesKeep = []
1133 if self.lite == False:
1134 for xm in crossmatches:
1135 group = groupedMatches[xm["merged_rank"] - 1]
1136 xm["merged_rank"] = None
1137 crossmatchesKeep.append(xm)
1139 if len(group) > 1:
1140 groupKeep = []
1141 uniqueIndexCheck = []
1142 for g in group:
1143 g["merged_rank"] = xm["rank"]
1144 g["rankScore"] = xm["rankScore"]
1145 index = "%(catalogue_table_name)s%(catalogue_object_id)s" % g
1146 # IF WE HAVE HIT A NEW SOURCE
1147 if index not in uniqueIndexCheck:
1148 uniqueIndexCheck.append(index)
1149 crossmatchesKeep.append(g)
1151 crossmatches = crossmatchesKeep
1153 self.log.debug('completed the ``_rank_classifications`` method')
1155 return classifications, crossmatches
1157 def _print_results_to_stdout(
1158 self,
1159 classifications,
1160 crossmatches):
1161 """*print the classification and crossmatch results for a single transient object to stdout*
1163 **Key Arguments**
1165 - ``crossmatches`` -- the unranked crossmatch classifications
1166 - ``classifications`` -- the classifications assigned to the transients post-crossmatches (dictionary of rank ordered list of classifications)
1169 .. todo ::
1171 - update key arguments values and definitions with defaults
1172 - update return values and definitions
1173 - update usage examples and text
1174 - update docstring text
1175 - check sublime snippet exists
1176 - clip any useful text to docs mindmap
1177 - regenerate the docs and check redendering of this docstring
1178 """
1179 self.log.debug('starting the ``_print_results_to_stdout`` method')
1181 if self.verbose == 0:
1182 return
1184 crossmatchesCopy = copy.deepcopy(crossmatches)
1186 # REPORT ONLY THE MOST PREFERED MAGNITUDE VALUE
1187 basic = ["association_type", "rank", "rankScore", "catalogue_table_name", "catalogue_object_id", "catalogue_object_type", "catalogue_object_subtype",
1188 "raDeg", "decDeg", "separationArcsec", "physical_separation_kpc", "direct_distance", "distance", "z", "photoZ", "photoZErr", "Mag", "MagFilter", "MagErr", "classificationReliability", "merged_rank"]
1189 verbose = ["search_name", "catalogue_view_name", "original_search_radius_arcsec", "direct_distance_modulus", "distance_modulus", "direct_distance_scale", "major_axis_arcsec", "scale", "U", "UErr",
1190 "B", "BErr", "V", "VErr", "R", "RErr", "I", "IErr", "J", "JErr", "H", "HErr", "K", "KErr", "_u", "_uErr", "_g", "_gErr", "_r", "_rErr", "_i", "_iErr", "_z", "_zErr", "_y", "G", "GErr", "_yErr", "unkMag"]
1191 dontFormat = ["decDeg", "raDeg", "rank",
1192 "catalogue_object_id", "catalogue_object_subtype", "merged_rank"]
1193 if self.verbose == 2:
1194 basic = basic + verbose
1196 for n in self.name:
1198 if n in classifications:
1199 headline = "\n" + n + "'s Predicted Classification: " + \
1200 classifications[n][0]
1201 else:
1202 headline = n + "'s Predicted Classification: ORPHAN"
1203 print(headline)
1204 print("Suggested Associations:")
1206 myCrossmatches = []
1207 myCrossmatches[:] = [c for c in crossmatchesCopy if c[
1208 "transient_object_id"] == n]
1210 for c in myCrossmatches:
1211 for f in self.filterPreference:
1212 if f in c and c[f]:
1213 c["Mag"] = c[f]
1214 c["MagFilter"] = f.replace("_", "").replace("Mag", "")
1215 if f + "Err" in c:
1216 c["MagErr"] = c[f + "Err"]
1217 else:
1218 c["MagErr"] = None
1219 break
1221 allKeys = []
1222 for c in myCrossmatches:
1223 for k, v in list(c.items()):
1224 if k not in allKeys:
1225 allKeys.append(k)
1227 for c in myCrossmatches:
1228 for k in allKeys:
1229 if k not in c:
1230 c[k] = None
1232 printCrossmatches = []
1233 for c in myCrossmatches:
1234 ordDict = collections.OrderedDict(sorted({}.items()))
1235 for k in basic:
1236 if k in c:
1237 if k == "catalogue_table_name":
1238 c[k] = c[k].replace(
1239 "tcs_cat_", "").replace("_", " ")
1240 if k == "classificationReliability":
1241 if c[k] == 1:
1242 c["classification reliability"] = "synonym"
1243 elif c[k] == 2:
1244 c["classification reliability"] = "association"
1245 elif c[k] == 3:
1246 c["classification reliability"] = "annotation"
1247 k = "classification reliability"
1248 if k == "catalogue_object_subtype" and "sdss" in c["catalogue_table_name"]:
1249 if c[k] == 6:
1250 c[k] = "galaxy"
1251 elif c[k] == 3:
1252 c[k] = "star"
1253 columnName = k.replace(
1254 "tcs_cat_", "").replace("_", " ")
1255 value = c[k]
1256 if k not in dontFormat:
1257 try:
1258 ordDict[columnName] = "%(value)0.3f" % locals()
1259 except:
1260 ordDict[columnName] = value
1261 else:
1262 ordDict[columnName] = value
1264 printCrossmatches.append(ordDict)
1266 outputFormat = None
1267 # outputFormat = "csv"
1269 from fundamentals.renderer import list_of_dictionaries
1270 dataSet = list_of_dictionaries(
1271 log=self.log,
1272 listOfDictionaries=printCrossmatches
1273 )
1275 if outputFormat == "csv":
1276 tableData = dataSet.csv(filepath=None)
1277 else:
1278 tableData = dataSet.table(filepath=None)
1280 print(tableData)
1282 self.log.debug('completed the ``_print_results_to_stdout`` method')
1283 return None
1285 def _lighten_return(
1286 self,
1287 crossmatches):
1288 """*lighten the classification and crossmatch results for smaller database footprint*
1290 **Key Arguments**
1292 - ``classifications`` -- the classifications assigned to the transients post-crossmatches (dictionary of rank ordered list of classifications)
1293 """
1294 self.log.debug('starting the ``_lighten_return`` method')
1296 # REPORT ONLY THE MOST PREFERED MAGNITUDE VALUE
1297 basic = ["transient_object_id", "association_type", "catalogue_table_name", "catalogue_object_id", "catalogue_object_type",
1298 "raDeg", "decDeg", "separationArcsec", "northSeparationArcsec", "eastSeparationArcsec", "physical_separation_kpc", "direct_distance", "distance", "z", "photoZ", "photoZErr", "Mag", "MagFilter", "MagErr", "classificationReliability", "major_axis_arcsec"]
1299 verbose = ["search_name", "catalogue_view_name", "original_search_radius_arcsec", "direct_distance_modulus", "distance_modulus", "direct_distance_scale", "scale", "U", "UErr",
1300 "B", "BErr", "V", "VErr", "R", "RErr", "I", "IErr", "J", "JErr", "H", "HErr", "K", "KErr", "_u", "_uErr", "_g", "_gErr", "_r", "_rErr", "_i", "_iErr", "_z", "_zErr", "_y", "G", "GErr", "_yErr", "unkMag"]
1301 dontFormat = ["decDeg", "raDeg", "rank",
1302 "catalogue_object_id", "catalogue_object_subtype", "merged_rank", "classificationReliability"]
1303 if self.verbose == 2:
1304 basic = basic + verbose
1306 for c in crossmatches:
1307 for f in self.filterPreference:
1308 if f in c and c[f]:
1309 c["Mag"] = c[f]
1310 c["MagFilter"] = f.replace("_", "").replace("Mag", "")
1311 if f + "Err" in c:
1312 c["MagErr"] = c[f + "Err"]
1313 else:
1314 c["MagErr"] = None
1315 break
1317 allKeys = []
1318 for c in crossmatches:
1319 for k, v in list(c.items()):
1320 if k not in allKeys:
1321 allKeys.append(k)
1323 for c in crossmatches:
1324 for k in allKeys:
1325 if k not in c:
1326 c[k] = None
1328 liteCrossmatches = []
1329 for c in crossmatches:
1330 ordDict = collections.OrderedDict(sorted({}.items()))
1331 for k in basic:
1332 if k in c:
1333 if k == "catalogue_table_name":
1334 c[k] = c[k].replace(
1335 "tcs_cat_", "").replace("_", " ")
1336 if k == "catalogue_object_subtype" and "sdss" in c["catalogue_table_name"]:
1337 if c[k] == 6:
1338 c[k] = "galaxy"
1339 elif c[k] == 3:
1340 c[k] = "star"
1341 columnName = k.replace(
1342 "tcs_cat_", "")
1343 value = c[k]
1344 if k not in dontFormat:
1345 try:
1346 ordDict[columnName] = float(f'{value:0.3f}')
1347 except:
1348 ordDict[columnName] = value
1349 else:
1350 ordDict[columnName] = value
1352 liteCrossmatches.append(ordDict)
1354 self.log.debug('completed the ``_lighten_return`` method')
1355 return liteCrossmatches
1357 def _consolidate_coordinateList(
1358 self,
1359 coordinateList):
1360 """*match the coordinate list against itself with the parameters of the NED search queries to minimise duplicated NED queries*
1362 **Key Arguments**
1364 - ``coordinateList`` -- the original coordinateList.
1367 **Return**
1369 - ``updatedCoordinateList`` -- the coordinate list with duplicated search areas removed
1372 **Usage**
1374 .. todo::
1376 - add usage info
1377 - create a sublime snippet for usage
1378 - update package tutorial if needed
1380 ```python
1381 usage code
1382 ```
1385 .. todo ::
1387 - update key arguments values and definitions with defaults
1388 - update return values and definitions
1389 - update usage examples and text
1390 - update docstring text
1391 - check sublime snippet exists
1392 - clip any useful text to docs mindmap
1393 - regenerate the docs and check redendering of this docstring
1394 """
1395 self.log.debug('starting the ``_consolidate_coordinateList`` method')
1397 raList = []
1398 raList[:] = np.array([c[0] for c in coordinateList])
1399 decList = []
1400 decList[:] = np.array([c[1] for c in coordinateList])
1402 nedStreamRadius = old_div(self.settings[
1403 "ned stream search radius arcec"], (60. * 60.))
1404 firstPassNedSearchRadius = old_div(self.settings[
1405 "first pass ned search radius arcec"], (60. * 60.))
1406 radius = nedStreamRadius - firstPassNedSearchRadius
1408 # LET'S BE CONSERVATIVE
1409 # radius = radius * 0.9
1411 xmatcher = sets(
1412 log=self.log,
1413 ra=raList,
1414 dec=decList,
1415 radius=radius, # in degrees
1416 sourceList=coordinateList,
1417 convertToArray=False
1418 )
1419 allMatches = xmatcher.match
1421 updatedCoordianteList = []
1422 for aSet in allMatches:
1423 updatedCoordianteList.append(aSet[0])
1425 self.log.debug('completed the ``_consolidate_coordinateList`` method')
1426 return updatedCoordianteList
1428 def classification_annotations(
1429 self):
1430 """*add a detialed classification annotation to each classification in the sherlock_classifications table*
1432 **Key Arguments**
1434 # -
1437 **Return**
1439 - None
1442 **Usage**
1444 .. todo::
1446 - add usage info
1447 - create a sublime snippet for usage
1448 - write a command-line tool for this method
1449 - update package tutorial with command-line tool info if needed
1451 ```python
1452 usage code
1453 ```
1456 .. todo ::
1458 - update key arguments values and definitions with defaults
1459 - update return values and definitions
1460 - update usage examples and text
1461 - update docstring text
1462 - check sublime snippet exists
1463 - clip any useful text to docs mindmap
1464 - regenerate the docs and check redendering of this docstring
1465 """
1466 self.log.debug('starting the ``classification_annotations`` method')
1468 from fundamentals.mysql import readquery
1469 sqlQuery = u"""
1470 select * from sherlock_classifications cl, sherlock_crossmatches xm where cl.transient_object_id=xm.transient_object_id and cl.annotation is null
1471 """ % locals()
1472 topXMs = readquery(
1473 log=self.log,
1474 sqlQuery=sqlQuery,
1475 dbConn=self.transientsDbConn
1476 )
1478 for xm in topXMs:
1479 annotation = []
1480 classType = xm["classificationReliability"]
1481 if classType == 1:
1482 annotation.append("is synonymous with")
1483 elif classType in [2, 3]:
1484 annotation.append("is possibly associated with")
1486 self.log.debug('completed the ``classification_annotations`` method')
1487 return None
1489 def update_classification_annotations_and_summaries(
1490 self,
1491 updatePeakMagnitudes=True,
1492 cl=False,
1493 crossmatches=False,
1494 classifications=False):
1495 """*update classification annotations and summaries*
1497 **Key Arguments**
1499 - ``updatePeakMagnitudes`` -- update the peak magnitudes in the annotations to give absolute magnitudes. Default *True*
1500 - ``cl`` -- reporting only to the command-line, do not update database. Default *False*
1501 - ``crossmatches`` -- crossmatches will be passed for the single classifications to report annotations from command-line
1502 - ``classifications`` -- classifications will be passed for the single classifications to have annotation appended to the dictionary for stand-alone non-database scripts
1504 **Return**
1506 - None
1509 **Usage**
1511 .. todo::
1513 - add usage info
1514 - create a sublime snippet for usage
1515 - write a command-line tool for this method
1516 - update package tutorial with command-line tool info if needed
1518 ```python
1519 usage code
1520 ```
1523 .. todo ::
1525 - update key arguments values and definitions with defaults
1526 - update return values and definitions
1527 - update usage examples and text
1528 - update docstring text
1529 - check sublime snippet exists
1530 - clip any useful text to docs mindmap
1531 - regenerate the docs and check redendering of this docstring
1532 """
1533 self.log.debug(
1534 'starting the ``update_classification_annotations_and_summaries`` method')
1536 # import time
1537 # start_time = time.time()
1538 # print "COLLECTING TRANSIENTS WITH NO ANNOTATIONS"
1540 # BULK RUN
1541 if crossmatches == False:
1542 if updatePeakMagnitudes:
1543 sqlQuery = u"""
1544 SELECT * from sherlock_crossmatches cm, sherlock_classifications cl where rank =1 and cl.transient_object_id= cm.transient_object_id and ((cl.classification not in ("AGN","CV","BS","VS") AND cm.dateLastModified > DATE_SUB(NOW(), INTERVAL 1 Day)) or cl.annotation is null)
1545 -- SELECT * from sherlock_crossmatches cm, sherlock_classifications cl where rank =1 and cl.transient_object_id= cm.transient_object_id and (cl.annotation is null or cl.dateLastModified is null or cl.dateLastModified > DATE_SUB(NOW(), INTERVAL 30 DAY)) order by cl.dateLastModified asc limit 100000
1546 """ % locals()
1547 else:
1548 sqlQuery = u"""
1549 SELECT * from sherlock_crossmatches cm, sherlock_classifications cl where rank =1 and cl.transient_object_id=cm.transient_object_id and cl.summary is null
1550 """ % locals()
1552 rows = readquery(
1553 log=self.log,
1554 sqlQuery=sqlQuery,
1555 dbConn=self.transientsDbConn
1556 )
1557 # COMMAND-LINE SINGLE CLASSIFICATION
1558 else:
1559 rows = crossmatches
1561 # print "FINISHED COLLECTING TRANSIENTS WITH NO ANNOTATIONS/GENERATING ANNOTATIONS: %d" % (time.time() - start_time,)
1562 # start_time = time.time()
1564 updates = []
1566 for row in rows:
1567 annotation, summary, sep = self.generate_match_annotation(
1568 match=row, updatePeakMagnitudes=updatePeakMagnitudes)
1570 if cl and "rank" in row and row["rank"] == 1:
1571 if classifications != False:
1572 classifications[
1573 row["transient_object_id"]].append(annotation)
1574 if self.verbose != 0:
1575 print("\n" + annotation)
1577 update = {
1578 "transient_object_id": row["transient_object_id"],
1579 "annotation": annotation,
1580 "summary": summary,
1581 "separationArcsec": sep
1582 }
1583 updates.append(update)
1585 if cl:
1586 return classifications
1588 # print "FINISHED GENERATING ANNOTATIONS/ADDING ANNOTATIONS TO TRANSIENT DATABASE: %d" % (time.time() - start_time,)
1589 # start_time = time.time()
1591 insert_list_of_dictionaries_into_database_tables(
1592 dbConn=self.transientsDbConn,
1593 log=self.log,
1594 dictList=updates,
1595 dbTableName="sherlock_classifications",
1596 dateModified=True,
1597 batchSize=10000,
1598 replace=True,
1599 dbSettings=self.settings["database settings"]["transients"]
1600 )
1602 # print "FINISHED ADDING ANNOTATIONS TO TRANSIENT DATABASE/UPDATING ORPHAN ANNOTATIONS: %d" % (time.time() - start_time,)
1603 # start_time = time.time()
1605 sqlQuery = """update sherlock_classifications set annotation = "The transient location is not matched against any known catalogued source", summary = "No catalogued match" where classification = 'ORPHAN' and summary is null """ % locals()
1606 writequery(
1607 log=self.log,
1608 sqlQuery=sqlQuery,
1609 dbConn=self.transientsDbConn,
1610 )
1612 # print "FINISHED UPDATING ORPHAN ANNOTATIONS: %d" % (time.time() - start_time,)
1613 # start_time = time.time()
1615 self.log.debug(
1616 'completed the ``update_classification_annotations_and_summaries`` method')
1617 return None
1619 # use the tab-trigger below for new method
1620 def update_peak_magnitudes(
1621 self):
1622 """*update peak magnitudes*
1624 **Key Arguments**
1626 # -
1629 **Return**
1631 - None
1634 **Usage**
1636 .. todo::
1638 - add usage info
1639 - create a sublime snippet for usage
1640 - write a command-line tool for this method
1641 - update package tutorial with command-line tool info if needed
1643 ```python
1644 usage code
1645 ```
1648 .. todo ::
1650 - update key arguments values and definitions with defaults
1651 - update return values and definitions
1652 - update usage examples and text
1653 - update docstring text
1654 - check sublime snippet exists
1655 - clip any useful text to docs mindmap
1656 - regenerate the docs and check redendering of this docstring
1657 """
1658 self.log.debug('starting the ``update_peak_magnitudes`` method')
1660 sqlQuery = self.settings["database settings"][
1661 "transients"]["transient peak magnitude query"]
1663 sqlQuery = """UPDATE sherlock_crossmatches s,
1664 (%(sqlQuery)s) t
1665 SET
1666 s.transientAbsMag = ROUND(t.mag - IFNULL(direct_distance_modulus,
1667 distance_modulus),
1668 2)
1669 WHERE
1670 IFNULL(direct_distance_modulus,
1671 distance_modulus) IS NOT NULL
1672 AND (s.association_type not in ("AGN","CV","BS","VS")
1673 or s.transientAbsMag is null)
1674 AND t.id = s.transient_object_id
1675 AND (s.dateLastModified > DATE_SUB(NOW(), INTERVAL 1 DAY));""" % locals()
1677 writequery(
1678 log=self.log,
1679 sqlQuery=sqlQuery,
1680 dbConn=self.transientsDbConn,
1681 )
1683 self.log.debug('completed the ``update_peak_magnitudes`` method')
1684 return None
1686 def _create_tables_if_not_exist(
1687 self):
1688 """*create the sherlock helper tables if they don't yet exist*
1690 **Key Arguments**
1692 # -
1695 **Return**
1697 - None
1700 **Usage**
1702 .. todo::
1704 - add usage info
1705 - create a sublime snippet for usage
1706 - write a command-line tool for this method
1707 - update package tutorial with command-line tool info if needed
1709 ```python
1710 usage code
1711 ```
1714 .. todo ::
1716 - update key arguments values and definitions with defaults
1717 - update return values and definitions
1718 - update usage examples and text
1719 - update docstring text
1720 - check sublime snippet exists
1721 - clip any useful text to docs mindmap
1722 - regenerate the docs and check redendering of this docstring
1723 """
1724 self.log.debug('starting the ``_create_tables_if_not_exist`` method')
1726 transientTable = self.settings["database settings"][
1727 "transients"]["transient table"]
1728 transientTableClassCol = self.settings["database settings"][
1729 "transients"]["transient classification column"]
1730 transientTableIdCol = self.settings["database settings"][
1731 "transients"]["transient primary id column"]
1733 crossmatchTable = "sherlock_crossmatches"
1734 createStatement = """
1735CREATE TABLE IF NOT EXISTS `sherlock_crossmatches` (
1736 `transient_object_id` bigint(20) unsigned DEFAULT NULL,
1737 `catalogue_object_id` varchar(200) COLLATE utf8_unicode_ci DEFAULT NULL,
1738 `catalogue_table_id` smallint(5) unsigned DEFAULT NULL,
1739 `separationArcsec` double DEFAULT NULL,
1740 `northSeparationArcsec` double DEFAULT NULL,
1741 `eastSeparationArcsec` double DEFAULT NULL,
1742 `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT,
1743 `z` double DEFAULT NULL,
1744 `scale` double DEFAULT NULL,
1745 `distance` double DEFAULT NULL,
1746 `distance_modulus` double DEFAULT NULL,
1747 `photoZ` double DEFAULT NULL,
1748 `photoZErr` double DEFAULT NULL,
1749 `association_type` varchar(45) COLLATE utf8_unicode_ci DEFAULT NULL,
1750 `dateCreated` datetime DEFAULT NULL,
1751 `physical_separation_kpc` double DEFAULT NULL,
1752 `catalogue_object_type` varchar(45) COLLATE utf8_unicode_ci DEFAULT NULL,
1753 `catalogue_object_subtype` varchar(45) COLLATE utf8_unicode_ci DEFAULT NULL,
1754 `association_rank` int(11) DEFAULT NULL,
1755 `catalogue_table_name` varchar(100) COLLATE utf8_unicode_ci DEFAULT NULL,
1756 `catalogue_view_name` varchar(100) COLLATE utf8_unicode_ci DEFAULT NULL,
1757 `rank` int(11) DEFAULT NULL,
1758 `rankScore` double DEFAULT NULL,
1759 `search_name` varchar(100) COLLATE utf8_unicode_ci DEFAULT NULL,
1760 `major_axis_arcsec` double DEFAULT NULL,
1761 `direct_distance` double DEFAULT NULL,
1762 `direct_distance_scale` double DEFAULT NULL,
1763 `direct_distance_modulus` double DEFAULT NULL,
1764 `raDeg` double DEFAULT NULL,
1765 `decDeg` double DEFAULT NULL,
1766 `original_search_radius_arcsec` double DEFAULT NULL,
1767 `catalogue_view_id` int(11) DEFAULT NULL,
1768 `U` double DEFAULT NULL,
1769 `UErr` double DEFAULT NULL,
1770 `B` double DEFAULT NULL,
1771 `BErr` double DEFAULT NULL,
1772 `V` double DEFAULT NULL,
1773 `VErr` double DEFAULT NULL,
1774 `R` double DEFAULT NULL,
1775 `RErr` double DEFAULT NULL,
1776 `I` double DEFAULT NULL,
1777 `IErr` double DEFAULT NULL,
1778 `J` double DEFAULT NULL,
1779 `JErr` double DEFAULT NULL,
1780 `H` double DEFAULT NULL,
1781 `HErr` double DEFAULT NULL,
1782 `K` double DEFAULT NULL,
1783 `KErr` double DEFAULT NULL,
1784 `_u` double DEFAULT NULL,
1785 `_uErr` double DEFAULT NULL,
1786 `_g` double DEFAULT NULL,
1787 `_gErr` double DEFAULT NULL,
1788 `_r` double DEFAULT NULL,
1789 `_rErr` double DEFAULT NULL,
1790 `_i` double DEFAULT NULL,
1791 `_iErr` double DEFAULT NULL,
1792 `_z` double DEFAULT NULL,
1793 `_zErr` double DEFAULT NULL,
1794 `_y` double DEFAULT NULL,
1795 `_yErr` double DEFAULT NULL,
1796 `G` double DEFAULT NULL,
1797 `GErr` double DEFAULT NULL,
1798 `W1` double DEFAULT NULL,
1799 `W1Err` double DEFAULT NULL,
1800 `unkMag` double DEFAULT NULL,
1801 `unkMagErr` double DEFAULT NULL,
1802 `dateLastModified` datetime DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
1803 `updated` tinyint(4) DEFAULT '0',
1804 `classificationReliability` tinyint(4) DEFAULT NULL,
1805 `transientAbsMag` double DEFAULT NULL,
1806 `merged_rank` tinyint(4) DEFAULT NULL,
1807 PRIMARY KEY (`id`),
1808 KEY `key_transient_object_id` (`transient_object_id`),
1809 KEY `key_catalogue_object_id` (`catalogue_object_id`),
1810 KEY `idx_separationArcsec` (`separationArcsec`),
1811 KEY `idx_rank` (`rank`)
1812) ENGINE=InnoDB AUTO_INCREMENT=0 DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci;
1814CREATE TABLE IF NOT EXISTS `sherlock_classifications` (
1815 `transient_object_id` bigint(20) NOT NULL,
1816 `classification` varchar(45) DEFAULT NULL,
1817 `annotation` TEXT COLLATE utf8_unicode_ci DEFAULT NULL,
1818 `summary` VARCHAR(50) COLLATE utf8_unicode_ci DEFAULT NULL,
1819 `separationArcsec` DOUBLE DEFAULT NULL,
1820 `matchVerified` TINYINT NULL DEFAULT NULL,
1821 `developmentComment` VARCHAR(100) NULL,
1822 `dateLastModified` datetime DEFAULT CURRENT_TIMESTAMP,
1823 `dateCreated` datetime DEFAULT CURRENT_TIMESTAMP,
1824 `updated` varchar(45) DEFAULT '0',
1825 PRIMARY KEY (`transient_object_id`),
1826 KEY `key_transient_object_id` (`transient_object_id`),
1827 KEY `idx_summary` (`summary`),
1828 KEY `idx_classification` (`classification`),
1829 KEY `idx_dateLastModified` (`dateLastModified`)
1830) ENGINE=MyISAM DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci;
1832""" % locals()
1834 # A FIX FOR MYSQL VERSIONS < 5.6
1835 triggers = []
1836 if float(self.dbVersions["transients"][:3]) < 5.6:
1837 createStatement = createStatement.replace(
1838 "`dateLastModified` datetime DEFAULT CURRENT_TIMESTAMP,", "`dateLastModified` datetime DEFAULT NULL,")
1839 createStatement = createStatement.replace(
1840 "`dateCreated` datetime DEFAULT CURRENT_TIMESTAMP,", "`dateCreated` datetime DEFAULT NULL,")
1842 triggers.append("""
1843CREATE TRIGGER dateCreated
1844BEFORE INSERT ON `%(crossmatchTable)s`
1845FOR EACH ROW
1846BEGIN
1847 IF NEW.dateCreated IS NULL THEN
1848 SET NEW.dateCreated = NOW();
1849 SET NEW.dateLastModified = NOW();
1850 END IF;
1851END""" % locals())
1853 try:
1854 writequery(
1855 log=self.log,
1856 sqlQuery=createStatement,
1857 dbConn=self.transientsDbConn,
1858 Force=True
1859 )
1860 except:
1861 self.log.info(
1862 "Could not create table (`%(crossmatchTable)s`). Probably already exist." % locals())
1864 sqlQuery = u"""
1865 SHOW TRIGGERS;
1866 """ % locals()
1867 rows = readquery(
1868 log=self.log,
1869 sqlQuery=sqlQuery,
1870 dbConn=self.transientsDbConn,
1871 )
1873 # DON'T ADD TRIGGERS IF THEY ALREADY EXIST
1874 for r in rows:
1875 if r["Trigger"] in ("sherlock_classifications_BEFORE_INSERT", "sherlock_classifications_AFTER_INSERT"):
1876 return None
1878 triggers.append("""CREATE TRIGGER `sherlock_classifications_BEFORE_INSERT` BEFORE INSERT ON `sherlock_classifications` FOR EACH ROW
1879BEGIN
1880 IF new.classification = "ORPHAN" THEN
1881 SET new.annotation = "The transient location is not matched against any known catalogued source", new.summary = "No catalogued match";
1882 END IF;
1883END""" % locals())
1885 triggers.append("""CREATE TRIGGER `sherlock_classifications_AFTER_INSERT` AFTER INSERT ON `sherlock_classifications` FOR EACH ROW
1886BEGIN
1887 update `%(transientTable)s` set `%(transientTableClassCol)s` = new.classification
1888 where `%(transientTableIdCol)s` = new.transient_object_id;
1889END""" % locals())
1891 for t in triggers:
1892 try:
1893 writequery(
1894 log=self.log,
1895 sqlQuery=t,
1896 dbConn=self.transientsDbConn,
1897 Force=True
1898 )
1899 except:
1900 self.log.info(
1901 "Could not create trigger (`%(crossmatchTable)s`). Probably already exist." % locals())
1903 self.log.debug('completed the ``_create_tables_if_not_exist`` method')
1904 return None
1906 # use the tab-trigger below for new method
1907 def generate_match_annotation(
1908 self,
1909 match,
1910 updatePeakMagnitudes=False):
1911 """*generate a human readale annotation for the transient-catalogue source match*
1913 **Key Arguments**
1915 - ``match`` -- the source crossmatched against the transient
1916 - ``updatePeakMagnitudes`` -- update the peak magnitudes in the annotations to give absolute magnitudes. Default *False*
1919 **Return**
1921 - None
1924 **Usage**
1928 ```python
1929 usage code
1930 ```
1932 ---
1934 ```eval_rst
1935 .. todo::
1937 - add usage info
1938 - create a sublime snippet for usage
1939 - write a command-line tool for this method
1940 - update package tutorial with command-line tool info if needed
1941 ```
1942 """
1943 self.log.debug('starting the ``generate_match_annotation`` method')
1945 if "catalogue_object_subtype" not in match:
1946 match["catalogue_object_subtype"] = None
1947 catalogue = match["catalogue_table_name"]
1948 objectId = match["catalogue_object_id"]
1949 objectType = match["catalogue_object_type"]
1950 objectSubtype = match["catalogue_object_subtype"]
1951 catalogueString = catalogue
1952 if catalogueString is None:
1953 badGuy = match["transient_object_id"]
1954 print(f"Issue with object {badGuy}")
1955 raise TypeError(f"Issue with object {badGuy}")
1956 if "catalogue" not in catalogueString.lower():
1957 catalogueString = catalogue + " catalogue"
1958 if "/" in catalogueString:
1959 catalogueString += "s"
1961 if "ned" in catalogue.lower():
1962 objectId = objectId.replace("+", "%2B")
1963 objectId = '''<a href="https://ned.ipac.caltech.edu/cgi-bin/objsearch?objname=%(objectId)s&extend=no&hconst=73&omegam=0.27&omegav=0.73&corr_z=1&out_csys=Equatorial&out_equinox=J2000.0&obj_sort=RA+or+Longitude&of=pre_text&zv_breaker=30000.0&list_limit=5&img_stamp=YES">%(objectId)s</a>''' % locals()
1964 elif "sdss" in catalogue.lower():
1965 objectId = "http://skyserver.sdss.org/dr12/en/tools/explore/Summary.aspx?id=%(objectId)s" % locals(
1966 )
1968 ra = self.converter.ra_decimal_to_sexegesimal(
1969 ra=match["raDeg"],
1970 delimiter=""
1971 )
1972 dec = self.converter.dec_decimal_to_sexegesimal(
1973 dec=match["decDeg"],
1974 delimiter=""
1975 )
1976 betterName = "SDSS J" + ra[0:9] + dec[0:9]
1977 objectId = '''<a href="%(objectId)s">%(betterName)s</a>''' % locals()
1978 elif "milliquas" in catalogue.lower():
1979 thisName = objectId
1980 objectId = objectId.replace(" ", "+")
1981 objectId = '''<a href="https://heasarc.gsfc.nasa.gov/db-perl/W3Browse/w3table.pl?popupFrom=Query+Results&tablehead=name%%3Dheasarc_milliquas%%26description%%3DMillion+Quasars+Catalog+%%28MILLIQUAS%%29%%2C+Version+4.8+%%2822+June+2016%%29%%26url%%3Dhttp%%3A%%2F%%2Fheasarc.gsfc.nasa.gov%%2FW3Browse%%2Fgalaxy-catalog%%2Fmilliquas.html%%26archive%%3DN%%26radius%%3D1%%26mission%%3DGALAXY+CATALOG%%26priority%%3D5%%26tabletype%%3DObject&dummy=Examples+of+query+constraints%%3A&varon=name&bparam_name=%%3D%%22%(objectId)s%%22&bparam_name%%3A%%3Aunit=+&bparam_name%%3A%%3Aformat=char25&varon=ra&bparam_ra=&bparam_ra%%3A%%3Aunit=degree&bparam_ra%%3A%%3Aformat=float8%%3A.5f&varon=dec&bparam_dec=&bparam_dec%%3A%%3Aunit=degree&bparam_dec%%3A%%3Aformat=float8%%3A.5f&varon=bmag&bparam_bmag=&bparam_bmag%%3A%%3Aunit=mag&bparam_bmag%%3A%%3Aformat=float8%%3A4.1f&varon=rmag&bparam_rmag=&bparam_rmag%%3A%%3Aunit=mag&bparam_rmag%%3A%%3Aformat=float8%%3A4.1f&varon=redshift&bparam_redshift=&bparam_redshift%%3A%%3Aunit=+&bparam_redshift%%3A%%3Aformat=float8%%3A6.3f&varon=radio_name&bparam_radio_name=&bparam_radio_name%%3A%%3Aunit=+&bparam_radio_name%%3A%%3Aformat=char22&varon=xray_name&bparam_xray_name=&bparam_xray_name%%3A%%3Aunit=+&bparam_xray_name%%3A%%3Aformat=char22&bparam_lii=&bparam_lii%%3A%%3Aunit=degree&bparam_lii%%3A%%3Aformat=float8%%3A.5f&bparam_bii=&bparam_bii%%3A%%3Aunit=degree&bparam_bii%%3A%%3Aformat=float8%%3A.5f&bparam_broad_type=&bparam_broad_type%%3A%%3Aunit=+&bparam_broad_type%%3A%%3Aformat=char4&bparam_optical_flag=&bparam_optical_flag%%3A%%3Aunit=+&bparam_optical_flag%%3A%%3Aformat=char3&bparam_red_psf_flag=&bparam_red_psf_flag%%3A%%3Aunit=+&bparam_red_psf_flag%%3A%%3Aformat=char1&bparam_blue_psf_flag=&bparam_blue_psf_flag%%3A%%3Aunit=+&bparam_blue_psf_flag%%3A%%3Aformat=char1&bparam_ref_name=&bparam_ref_name%%3A%%3Aunit=+&bparam_ref_name%%3A%%3Aformat=char6&bparam_ref_redshift=&bparam_ref_redshift%%3A%%3Aunit=+&bparam_ref_redshift%%3A%%3Aformat=char6&bparam_qso_prob=&bparam_qso_prob%%3A%%3Aunit=percent&bparam_qso_prob%%3A%%3Aformat=int2%%3A3d&bparam_alt_name_1=&bparam_alt_name_1%%3A%%3Aunit=+&bparam_alt_name_1%%3A%%3Aformat=char22&bparam_alt_name_2=&bparam_alt_name_2%%3A%%3Aunit=+&bparam_alt_name_2%%3A%%3Aformat=char22&Entry=&Coordinates=J2000&Radius=Default&Radius_unit=arcsec&NR=CheckCaches%%2FGRB%%2FSIMBAD%%2BSesame%%2FNED&Time=&ResultMax=1000&displaymode=Display&Action=Start+Search&table=heasarc_milliquas">%(thisName)s</a>''' % locals()
1983 if objectSubtype and str(objectSubtype).lower() in ["uvs", "radios", "xray", "qso", "irs", 'uves', 'viss', 'hii', 'gclstr', 'ggroup', 'gpair', 'gtrpl']:
1984 objectType = objectSubtype
1986 if objectType == "star":
1987 objectType = "stellar source"
1988 elif objectType == "agn":
1989 objectType = "AGN"
1990 elif objectType == "cb":
1991 objectType = "CV"
1992 elif objectType == "unknown":
1993 objectType = "unclassified source"
1995 sep = match["separationArcsec"]
1996 if match["classificationReliability"] == 1:
1997 classificationReliability = "synonymous"
1998 psep = match["physical_separation_kpc"]
1999 if psep:
2000 location = '%(sep)0.1f" (%(psep)0.1f Kpc) from the %(objectType)s core' % locals(
2001 )
2002 else:
2003 location = '%(sep)0.1f" from the %(objectType)s core' % locals(
2004 )
2005 else:
2006 # elif match["classificationReliability"] in (2, 3):
2007 classificationReliability = "possibly associated"
2008 n = float(match["northSeparationArcsec"])
2009 if n > 0:
2010 nd = "S"
2011 else:
2012 nd = "N"
2013 e = float(match["eastSeparationArcsec"])
2014 if e > 0:
2015 ed = "W"
2016 else:
2017 ed = "E"
2018 n = math.fabs(float(n))
2019 e = math.fabs(float(e))
2020 psep = match["physical_separation_kpc"]
2021 if psep:
2022 location = '%(n)0.2f" %(nd)s, %(e)0.2f" %(ed)s (%(psep)0.1f Kpc) from the %(objectType)s centre' % locals(
2023 )
2024 else:
2025 location = '%(n)0.2f" %(nd)s, %(e)0.2f" %(ed)s from the %(objectType)s centre' % locals(
2026 )
2027 location = location.replace("unclassified", "object's")
2029 best_mag = None
2030 best_mag_error = None
2031 best_mag_filter = None
2032 filters = ["R", "V", "B", "I", "J", "G", "H", "K", "U",
2033 "_r", "_g", "_i", "_g", "_z", "_y", "_u", "W1", "unkMag"]
2034 for f in filters:
2035 if f in match and match[f] and not best_mag:
2036 best_mag = match[f]
2037 try:
2038 best_mag_error = match[f + "Err"]
2039 except:
2040 pass
2041 subfilter = f.replace(
2042 "_", "").replace("Mag", "")
2043 best_mag_filter = f.replace(
2044 "_", "").replace("Mag", "") + "="
2045 if "unk" in best_mag_filter:
2046 best_mag_filter = ""
2047 subfilter = ''
2049 if not best_mag_filter:
2050 if str(best_mag).lower() in ("8", "11", "18"):
2051 best_mag_filter = "an "
2052 else:
2053 best_mag_filter = "a "
2054 else:
2055 if str(best_mag_filter)[0].lower() in ("r", "i", "h"):
2056 best_mag_filter = "an " + best_mag_filter
2057 else:
2058 best_mag_filter = "a " + best_mag_filter
2059 if not best_mag:
2060 best_mag = "an unknown-"
2061 best_mag_filter = ""
2062 else:
2063 best_mag = "%(best_mag)0.2f " % locals()
2065 distance = None
2066 if "direct_distance" in match and match["direct_distance"]:
2067 d = match["direct_distance"]
2068 distance = "distance of %(d)0.1f Mpc" % locals()
2070 if match["z"]:
2071 z = match["z"]
2072 distance += "(z=%(z)0.3f)" % locals()
2073 elif "z" in match and match["z"]:
2074 z = match["z"]
2075 distance = "z=%(z)0.3f" % locals()
2076 elif "photoZ" in match and match["photoZ"]:
2077 z = match["photoZ"]
2078 zErr = match["photoZErr"]
2079 if not zErr:
2080 distance = "photoZ=%(z)0.3f" % locals()
2081 else:
2082 distance = "photoZ=%(z)0.3f (±%(zErr)0.3f)" % locals()
2084 if distance:
2085 distance = "%(distance)s" % locals()
2087 distance_modulus = None
2088 if match["direct_distance_modulus"]:
2089 distance_modulus = match["direct_distance_modulus"]
2090 elif match["distance_modulus"]:
2091 distance_modulus = match["distance_modulus"]
2093 if updatePeakMagnitudes:
2094 if distance:
2095 absMag = match["transientAbsMag"]
2096 absMag = """ A host %(distance)s implies a transient <em>M =</em> %(absMag)s mag.""" % locals(
2097 )
2098 else:
2099 absMag = ""
2100 else:
2101 if distance and distance_modulus:
2102 absMag = "%(distance_modulus)0.2f" % locals()
2103 absMag = """ A host %(distance)s implies a <em>m - M =</em> %(absMag)s.""" % locals(
2104 )
2105 else:
2106 absMag = ""
2108 annotation = "The transient is %(classificationReliability)s with <em>%(objectId)s</em>; %(best_mag_filter)s%(best_mag)smag %(objectType)s found in the %(catalogueString)s. It's located %(location)s.%(absMag)s" % locals()
2109 try:
2110 summary = '%(sep)0.1f" from %(objectType)s in %(catalogue)s' % locals()
2111 except:
2112 badGuy = match["transient_object_id"]
2113 print(f"Issue with object {badGuy}")
2114 raise TypeError(f"Issue with object {badGuy}")
2116 self.log.debug('completed the ``generate_match_annotation`` method')
2117 return annotation, summary, sep
2119 # use the tab-trigger below for new method
2120 # xt-class-method
2123def _crossmatch_transients_against_catalogues(
2124 transientsMetadataListIndex,
2125 log,
2126 settings,
2127 colMaps):
2128 """run the transients through the crossmatch algorithm in the settings file
2130 **Key Arguments**
2133 - ``transientsMetadataListIndex`` -- the list of transient metadata lifted from the database.
2134 - ``colMaps`` -- dictionary of dictionaries with the name of the database-view (e.g. `tcs_view_agn_milliquas_v4_5`) as the key and the column-name dictary map as value (`{view_name: {columnMap}}`).
2136 **Return**
2138 - ``crossmatches`` -- a list of dictionaries of the associated sources crossmatched from the catalogues database
2141 .. todo ::
2143 - update key arguments values and definitions with defaults
2144 - update return values and definitions
2145 - update usage examples and text
2146 - update docstring text
2147 - check sublime snippet exists
2148 - clip any useful text to docs mindmap
2149 - regenerate the docs and check redendering of this docstring
2150 """
2152 from fundamentals.mysql import database
2153 from sherlock import transient_catalogue_crossmatch
2155 global theseBatches
2157 log.debug(
2158 'starting the ``_crossmatch_transients_against_catalogues`` method')
2160 # SETUP ALL DATABASE CONNECTIONS
2162 transientsMetadataList = theseBatches[transientsMetadataListIndex]
2164 dbConn = database(
2165 log=log,
2166 dbSettings=settings["database settings"]["static catalogues"]
2167 ).connect()
2169 cm = transient_catalogue_crossmatch(
2170 log=log,
2171 dbConn=dbConn,
2172 transients=transientsMetadataList,
2173 settings=settings,
2174 colMaps=colMaps
2175 )
2176 crossmatches = cm.match()
2178 log.debug(
2179 'completed the ``_crossmatch_transients_against_catalogues`` method')
2181 return crossmatches