Coverage for sherlock/imports/ned_d.py: 12%
254 statements
« prev ^ index » next coverage.py v7.2.2, created at 2023-10-10 13:58 +0000
« prev ^ index » next coverage.py v7.2.2, created at 2023-10-10 13:58 +0000
1#!/usr/local/bin/python
2# encoding: utf-8
3"""
4*Import ned_d catalogue into sherlock-catalogues database*
6:Author:
7 David Young
8"""
9from __future__ import print_function
10from __future__ import division
11from ._base_importer import _base_importer
12from docopt import docopt
13from neddy import namesearch
14from sloancone import check_coverage
15from astrocalc.coords import unit_conversion
16from fundamentals.mysql import writequery, readquery
17import re
18import string
19import codecs
20import pickle
21import glob
22import time
23import csv
24import readline
26from builtins import zip
27from past.utils import old_div
28import sys
29import os
30os.environ['TERM'] = 'vt100'
33class ned_d(_base_importer):
34 """
35 *Import the * `NED-D <https://ned.ipac.caltech.edu/Library/Distances/>`_ *galaxy catalogue in to the sherlock-catalogues database*
37 **Key Arguments**
39 - ``log`` -- logger
40 - ``settings`` -- the settings dictionary
41 - ``pathToDataFile`` -- path to the ned_d data file
42 - ``version`` -- version of the ned_d catalogue
43 - ``catalogueName`` -- the name of the catalogue
46 **Usage**
48 To import the ned_d catalogue catalogue, run the following:
50 ```python
51 from sherlock.imports import ned_d
52 catalogue = ned_d(
53 log=log,
54 settings=settings,
55 pathToDataFile="/path/to/ned_d.txt",
56 version="1.0",
57 catalogueName="ned_d"
58 )
59 catalogue.ingest()
60 ```
63 .. todo ::
65 - abstract this module out into its own stand alone script
66 """
67 # INITIALIZATION
69 def ingest(self):
70 """Import the ned_d catalogue into the catalogues database
72 The method first generates a list of python dictionaries from the ned_d datafile, imports this list of dictionaries into a database table and then generates the HTMIDs for that table.
74 **Usage**
76 See class docstring for usage
79 .. todo ::
81 - update docstring text
82 - check sublime snippet exists
83 - clip any useful text to docs mindmap
84 - regenerate the docs and check redendering of this docstring
85 """
86 self.log.debug('starting the ``get`` method')
88 dictList = self._create_dictionary_of_ned_d()
89 self.primaryIdColumnName = "primaryId"
90 self.raColName = "raDeg"
91 self.declColName = "decDeg"
93 tableName = self.dbTableName
94 viewName = tableName.replace("tcs_cat_", "tcs_view_galaxy_")
95 createStatement = u"""
96 CREATE TABLE `%(tableName)s` (
97 `primaryId` bigint(20) NOT NULL AUTO_INCREMENT COMMENT 'An internal counter',
98 `Method` varchar(150) DEFAULT NULL,
99 `dateCreated` datetime DEFAULT CURRENT_TIMESTAMP,
100 `dateLastModified` datetime DEFAULT CURRENT_TIMESTAMP,
101 `updated` varchar(45) DEFAULT '0',
102 `dist_derived_from_sn` varchar(150) DEFAULT NULL,
103 `dist_in_ned_flag` varchar(10) DEFAULT NULL,
104 `dist_index_id` mediumint(9) DEFAULT NULL,
105 `dist_mod` double DEFAULT NULL,
106 `dist_mod_err` double DEFAULT NULL,
107 `dist_mpc` double DEFAULT NULL,
108 `galaxy_index_id` mediumint(9) DEFAULT NULL,
109 `hubble_const` double DEFAULT NULL,
110 `lmc_mod` double DEFAULT NULL,
111 `notes` varchar(500) DEFAULT NULL,
112 `primary_ned_id` varchar(150) DEFAULT NULL,
113 `redshift` double DEFAULT NULL,
114 `ref` varchar(150) DEFAULT NULL,
115 `ref_date` int(11) DEFAULT NULL,
116 `master_row` tinyint(4) DEFAULT '0',
117 `major_diameter_arcmin` double DEFAULT NULL,
118 `ned_notes` varchar(700) DEFAULT NULL,
119 `object_type` varchar(100) DEFAULT NULL,
120 `redshift_err` double DEFAULT NULL,
121 `redshift_quality` varchar(100) DEFAULT NULL,
122 `magnitude_filter` varchar(10) DEFAULT NULL,
123 `minor_diameter_arcmin` double DEFAULT NULL,
124 `morphology` varchar(50) DEFAULT NULL,
125 `hierarchy` varchar(50) DEFAULT NULL,
126 `galaxy_morphology` varchar(50) DEFAULT NULL,
127 `radio_morphology` varchar(50) DEFAULT NULL,
128 `activity_type` varchar(50) DEFAULT NULL,
129 `in_ned` tinyint(4) DEFAULT NULL,
130 `raDeg` double DEFAULT NULL,
131 `decDeg` double DEFAULT NULL,
132 `eb_v` double DEFAULT NULL,
133 `sdss_coverage` tinyint(4) DEFAULT NULL,
134 `htm16ID` bigint(20) DEFAULT NULL,
135 `htm13ID` int(11) DEFAULT NULL,
136 `htm10ID` int(11) DEFAULT NULL,
137 `magnitude` double DEFAULT NULL,
138 PRIMARY KEY (`primaryId`),
139 UNIQUE KEY `galaxy_index_id_dist_index_id` (`galaxy_index_id`,`dist_index_id`),
140 KEY `idx_htm10ID` (`htm10ID`),
141 KEY `idx_htm13ID` (`htm13ID`),
142 KEY `idx_htm16ID` (`htm16ID`),
143 KEY `idx_dist` (`dist_mpc`),
144 KEY `idx_redshift` (`redshift`),
145 KEY `idx_major_diameter` (`major_diameter_arcmin`),
146 KEY `idx_master_name` (`master_row`),
147 KEY `idx_magnitude` (`magnitude`),
148 KEY `idx_in_ned` (`in_ned`),
149 KEY `idx_type` (`object_type`),
150 KEY `idx_dist_from_sn` (`dist_derived_from_sn`)
151 ) ENGINE=MyISAM AUTO_INCREMENT=0 DEFAULT CHARSET=latin1;
152 DROP VIEW IF EXISTS `%(viewName)s`;
153 CREATE
154 VIEW `%(viewName)s` AS
155 (SELECT
156 `%(tableName)s`.`primaryId` AS `primaryId`,
157 `%(tableName)s`.`Method` AS `Method`,
158 `%(tableName)s`.`dateCreated` AS `dateCreated`,
159 `%(tableName)s`.`dist_derived_from_sn` AS `dist_derived_from_sn`,
160 `%(tableName)s`.`dist_in_ned_flag` AS `dist_in_ned_flag`,
161 `%(tableName)s`.`dist_index_id` AS `dist_index_id`,
162 `%(tableName)s`.`dist_mod_median` AS `dist_mod`,
163 `%(tableName)s`.`dist_mod_err` AS `dist_mod_err`,
164 `%(tableName)s`.`dist_mpc_median` AS `dist_mpc`,
165 `%(tableName)s`.`galaxy_index_id` AS `galaxy_index_id`,
166 `%(tableName)s`.`hubble_const` AS `hubble_const`,
167 `%(tableName)s`.`lmc_mod` AS `lmc_mod`,
168 `%(tableName)s`.`notes` AS `notes`,
169 `%(tableName)s`.`primary_ned_id` AS `primary_ned_id`,
170 `%(tableName)s`.`redshift` AS `redshift`,
171 `%(tableName)s`.`ref` AS `ref`,
172 `%(tableName)s`.`ref_date` AS `ref_date`,
173 `%(tableName)s`.`master_row` AS `master_row`,
174 `%(tableName)s`.`major_diameter_arcmin` AS `major_diameter_arcmin`,
175 `%(tableName)s`.`ned_notes` AS `ned_notes`,
176 `%(tableName)s`.`object_type` AS `object_type`,
177 `%(tableName)s`.`redshift_err` AS `redshift_err`,
178 `%(tableName)s`.`redshift_quality` AS `redshift_quality`,
179 `%(tableName)s`.`magnitude` AS `magnitude`,
180 `%(tableName)s`.`minor_diameter_arcmin` AS `minor_diameter_arcmin`,
181 `%(tableName)s`.`morphology` AS `morphology`,
182 `%(tableName)s`.`hierarchy` AS `hierarchy`,
183 `%(tableName)s`.`galaxy_morphology` AS `galaxy_morphology`,
184 `%(tableName)s`.`radio_morphology` AS `radio_morphology`,
185 `%(tableName)s`.`activity_type` AS `activity_type`,
186 `%(tableName)s`.`in_ned` AS `in_ned`,
187 `%(tableName)s`.`raDeg` AS `raDeg`,
188 `%(tableName)s`.`decDeg` AS `decDeg`,
189 `%(tableName)s`.`eb_v` AS `eb_v`,
190 `%(tableName)s`.`htm16ID` AS `htm16ID`,
191 `%(tableName)s`.`htm10ID` AS `htm10ID`,
192 `%(tableName)s`.`htm13ID` AS `htm13ID`,
193 `%(tableName)s`.`sdss_coverage` AS `sdss_coverage`
194 FROM
195 `%(tableName)s`
196 WHERE
197 `%(tableName)s`.`master_row` = 1);
198 """ % locals()
200 self.add_data_to_database_table(
201 dictList=dictList,
202 createStatement=createStatement
203 )
205 self._clean_up_columns()
206 self._get_metadata_for_galaxies()
207 # self._update_sdss_coverage()
209 self.log.debug('completed the ``get`` method')
210 return None
212 def _create_dictionary_of_ned_d(
213 self):
214 """create a list of dictionaries containing all the rows in the ned_d catalogue
216 **Return**
218 - ``dictList`` - a list of dictionaries containing all the rows in the ned_d catalogue
221 .. todo ::
223 - update key arguments values and definitions with defaults
224 - update return values and definitions
225 - update usage examples and text
226 - update docstring text
227 - check sublime snippet exists
228 - clip any useful text to docs mindmap
229 - regenerate the docs and check redendering of this docstring
230 """
231 self.log.debug(
232 'starting the ``_create_dictionary_of_ned_d`` method')
234 import pandas as pd
236 count = 0
237 with open(self.pathToDataFile, 'r') as csvFile:
238 csvReader = csv.reader(
239 csvFile, dialect='excel', delimiter=',', quotechar='"')
240 totalRows = sum(1 for row in csvReader)
241 csvFile.close()
242 totalCount = totalRows
244 with open(self.pathToDataFile, 'r') as csvFile:
245 csvReader = csv.reader(
246 csvFile, dialect='excel', delimiter=',', quotechar='"')
247 theseKeys = []
248 dictList = []
249 for row in csvReader:
250 if len(theseKeys) == 0:
251 totalRows -= 1
252 if "Exclusion Code" in row and "Hubble const." in row:
253 for i in row:
254 if i == "redshift (z)":
255 theseKeys.append("redshift")
256 elif i == "Hubble const.":
257 theseKeys.append("hubble_const")
258 elif i == "G":
259 theseKeys.append("galaxy_index_id")
260 elif i == "err":
261 theseKeys.append("dist_mod_err")
262 elif i == "D (Mpc)":
263 theseKeys.append("dist_mpc")
264 elif i == "Date (Yr. - 1980)":
265 theseKeys.append("ref_date")
266 elif i == "REFCODE":
267 theseKeys.append("ref")
268 elif i == "Exclusion Code":
269 theseKeys.append("dist_in_ned_flag")
270 elif i == "Adopted LMC modulus":
271 theseKeys.append("lmc_mod")
272 elif i == "m-M":
273 theseKeys.append("dist_mod")
274 elif i == "Notes":
275 theseKeys.append("notes")
276 elif i == "SN ID":
277 theseKeys.append("dist_derived_from_sn")
278 elif i == "method":
279 theseKeys.append("dist_method")
280 elif i == "Galaxy ID":
281 theseKeys.append("primary_ned_id")
282 elif i == "D":
283 theseKeys.append("dist_index_id")
284 else:
285 theseKeys.append(i)
286 continue
288 if len(theseKeys):
289 count += 1
290 if count > 1:
291 # Cursor up one line and clear line
292 sys.stdout.write("\x1b[1A\x1b[2K")
293 if count > totalCount:
294 count = totalCount
295 percent = (old_div(float(count), float(totalCount))) * 100.
296 print(
297 "%(count)s / %(totalCount)s (%(percent)1.1f%%) rows added to memory" % locals())
298 rowDict = {}
299 for t, r in zip(theseKeys, row):
300 rowDict[t] = r
301 if t == "ref_date":
302 try:
303 rowDict[t] = int(r) + 1980
304 except:
305 rowDict[t] = None
307 if rowDict["dist_index_id"] != "999999":
308 dictList.append(rowDict)
310 csvFile.close()
312 df = pd.DataFrame(dictList)
313 df['galaxy_index_id'] = df['galaxy_index_id'].values.astype(int)
314 # GROUP RESULTS
315 dfGroups = df.groupby(['galaxy_index_id'])
317 medianDistDf = dfGroups[["dist_mpc", "dist_mod"]].median().reset_index()
318 # RENAME COLUMNS
319 medianDistDf.rename(columns={"dist_mpc": "dist_mpc_median"}, inplace=True)
320 medianDistDf.rename(columns={"dist_mod": "dist_mod_median"}, inplace=True)
322 # MERGE DATAFRAMES
323 df = df.merge(medianDistDf, on=['galaxy_index_id'], how='outer')
325 # SORT BY COLUMN NAME
326 df.sort_values(['galaxy_index_id'],
327 ascending=[True], inplace=True)
329 dictList = df.to_dict('records')
330 self.log.debug(
331 'completed the ``_create_dictionary_of_ned_d`` method')
332 return dictList
334 def _clean_up_columns(
335 self):
336 """clean up columns of the NED table
338 .. todo ::
340 - update key arguments values and definitions with defaults
341 - update return values and definitions
342 - update usage examples and text
343 - update docstring text
344 - check sublime snippet exists
345 - clip any useful text to docs mindmap
346 - regenerate the docs and check redendering of this docstring
347 """
348 self.log.debug('starting the ``_clean_up_columns`` method')
350 tableName = self.dbTableName
352 print("cleaning up %(tableName)s columns" % locals())
354 sqlQuery = u"""
355 set sql_mode="STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION";
356 """ % locals()
357 writequery(
358 log=self.log,
359 sqlQuery=sqlQuery,
360 dbConn=self.cataloguesDbConn,
361 )
363 sqlQuery = u"""
364 update %(tableName)s set dist_mod_err = null where dist_mod_err = 0;
365 update %(tableName)s set dist_in_ned_flag = null where dist_in_ned_flag = "";
366 update %(tableName)s set notes = null where notes = "";
367 update %(tableName)s set redshift = null where redshift = 0;
368 update %(tableName)s set dist_derived_from_sn = null where dist_derived_from_sn = "";
369 update %(tableName)s set hubble_const = null where hubble_const = 0;
370 update %(tableName)s set lmc_mod = null where lmc_mod = 0;
371 update %(tableName)s set master_row = 0;
372 update %(tableName)s set master_row = 1 where primaryId in (select * from (select distinct primaryId from %(tableName)s group by galaxy_index_id) as alias);
373 """ % locals()
374 writequery(
375 log=self.log,
376 sqlQuery=sqlQuery,
377 dbConn=self.cataloguesDbConn,
378 )
380 self.log.debug('completed the ``_clean_up_columns`` method')
381 return None
383 def _get_metadata_for_galaxies(
384 self):
385 """get metadata for galaxies
387 .. todo ::
389 - update key arguments values and definitions with defaults
390 - update return values and definitions
391 - update usage examples and text
392 - update docstring text
393 - check sublime snippet exists
394 - clip any useful text to docs mindmap
395 - regenerate the docs and check redendering of this docstring
396 """
397 self.log.debug('starting the ``_get_metadata_for_galaxies`` method')
399 total, batches = self._count_galaxies_requiring_metadata()
400 print("%(total)s galaxies require metadata. Need to send %(batches)s batch requests to NED." % locals())
402 totalBatches = self.batches
403 thisCount = 0
405 # FOR EACH BATCH, GET THE GALAXY IDs, QUERY NED AND UPDATE THE DATABASE
406 while self.total:
407 thisCount += 1
408 self._get_3000_galaxies_needing_metadata()
409 dictList = self._query_ned_and_add_results_to_database(thisCount)
411 self.add_data_to_database_table(
412 dictList=dictList,
413 createStatement=False
414 )
416 self._count_galaxies_requiring_metadata()
418 self.log.debug('completed the ``_get_metadata_for_galaxies`` method')
419 return None
421 def _count_galaxies_requiring_metadata(
422 self):
423 """ count galaxies requiring metadata
425 **Return**
427 - ``self.total``, ``self.batches`` -- total number of galaxies needing metadata & the number of batches required to be sent to NED
430 .. todo ::
432 - update key arguments values and definitions with defaults
433 - update return values and definitions
434 - update usage examples and text
435 - update docstring text
436 - check sublime snippet exists
437 - clip any useful text to docs mindmap
438 - regenerate the docs and check redendering of this docstring
439 """
440 self.log.debug(
441 'starting the ``_count_galaxies_requiring_metadata`` method')
443 tableName = self.dbTableName
445 sqlQuery = u"""
446 select count(*) as count from %(tableName)s where master_row = 1 and in_ned is null
447 """ % locals()
448 rows = readquery(
449 log=self.log,
450 sqlQuery=sqlQuery,
451 dbConn=self.cataloguesDbConn,
452 quiet=False
453 )
454 self.total = rows[0]["count"]
455 self.batches = int(old_div(self.total, 3000.)) + 1
457 if self.total == 0:
458 self.batches = 0
460 self.log.debug(
461 'completed the ``_count_galaxies_requiring_metadata`` method')
462 return self.total, self.batches
464 def _get_3000_galaxies_needing_metadata(
465 self):
466 """ get 3000 galaxies needing metadata
468 **Return**
470 - ``len(self.theseIds)`` -- the number of NED IDs returned
473 .. todo ::
475 - update key arguments values and definitions with defaults
476 - update return values and definitions
477 - update usage examples and text
478 - update docstring text
479 - check sublime snippet exists
480 - clip any useful text to docs mindmap
481 - regenerate the docs and check redendering of this docstring
482 """
483 self.log.debug(
484 'starting the ``_get_3000_galaxies_needing_metadata`` method')
486 tableName = self.dbTableName
488 # SELECT THE DATA FROM NED TABLE
489 self.theseIds = {}
490 sqlQuery = u"""
491 select primaryId, primary_ned_id from %(tableName)s where master_row = 1 and in_ned is null limit 3000;
492 """ % locals()
493 rows = readquery(
494 log=self.log,
495 sqlQuery=sqlQuery,
496 dbConn=self.cataloguesDbConn,
497 quiet=False
498 )
499 for row in rows:
500 self.theseIds[row["primary_ned_id"]] = row["primaryId"]
502 self.log.debug(
503 'completed the ``_get_3000_galaxies_needing_metadata`` method')
505 return len(self.theseIds)
507 def _query_ned_and_add_results_to_database(
508 self,
509 batchCount):
510 """ query ned and add results to database
512 **Key Arguments**
514 - ``batchCount`` - the index number of the batch sent to NED
517 .. todo ::
519 - update key arguments values and definitions with defaults
520 - update return values and definitions
521 - update usage examples and text
522 - update docstring text
523 - check sublime snippet exists
524 - clip any useful text to docs mindmap
525 - regenerate the docs and check redendering of this docstring
526 """
527 self.log.debug(
528 'starting the ``_query_ned_and_add_results_to_database`` method')
530 import re
531 regex = re.compile(r'[^\d\.]')
533 tableName = self.dbTableName
534 # ASTROCALC UNIT CONVERTER OBJECT
535 converter = unit_conversion(
536 log=self.log
537 )
539 # QUERY NED WITH BATCH
540 totalCount = len(self.theseIds)
541 print("requesting metadata from NED for %(totalCount)s galaxies (batch %(batchCount)s)" % locals())
542 search = namesearch(
543 log=self.log,
544 names=list(self.theseIds.keys()),
545 quiet=True
546 )
547 results = search.get()
548 print("results returned from ned -- starting to add to database" % locals())
550 # CLEAN THE RETURNED DATA AND UPDATE DATABASE
551 totalCount = len(results)
552 count = 0
553 sqlQuery = ""
554 dictList = []
556 colList = ["redshift_quality", "redshift", "hierarchy", "object_type", "major_diameter_arcmin", "morphology", "magnitude_filter",
557 "ned_notes", "eb_v", "raDeg", "radio_morphology", "activity_type", "minor_diameter_arcmin", "decDeg", "redshift_err", "in_ned", "magnitude"]
559 if not len(results):
560 for k, v in list(self.theseIds.items()):
561 dictList.append({
562 "in_ned": 0,
563 "primaryID": v
564 })
565 for thisDict in results:
566 thisDict["magnitude"] = thisDict["magnitude_filter"]
567 thisDict["tableName"] = tableName
568 count += 1
569 for k, v in list(thisDict.items()):
570 if not v or len(v) == 0:
571 thisDict[k] = "null"
572 if k in ["major_diameter_arcmin", "minor_diameter_arcmin"] and (":" in v or "?" in v or "<" in v):
573 thisDict[k] = v.replace(":", "").replace(
574 "?", "").replace("<", "")
575 if isinstance(v, ("".__class__, u"".__class__)) and '"' in v:
576 thisDict[k] = v.replace('"', '\\"')
577 if k in ["magnitude"] and thisDict[k] != "null":
578 thisDict[k] = regex.sub("", thisDict[k])
580 if "Input name not" not in thisDict["input_note"] and "Same object as" not in thisDict["input_note"]:
581 if thisDict["ra"] != "null" and thisDict["dec"] != "null":
582 thisDict["raDeg"] = converter.ra_sexegesimal_to_decimal(
583 ra=thisDict["ra"]
584 )
585 thisDict["decDeg"] = converter.dec_sexegesimal_to_decimal(
586 dec=thisDict["dec"]
587 )
588 else:
589 thisDict["raDeg"] = None
590 thisDict["decDeg"] = None
591 thisDict["in_ned"] = 1
592 thisDict["eb_v"] = thisDict["eb-v"]
594 row = {}
595 row["primary_ned_id"] = thisDict["input_name"]
597 try:
598 row["primaryID"] = self.theseIds[thisDict["input_name"]]
599 for c in colList:
600 if thisDict[c] == "null":
601 row[c] = None
602 else:
603 row[c] = thisDict[c]
604 dictList.append(row)
605 except:
606 g = thisDict["input_name"]
607 self.log.error(
608 "Cannot find database table %(tableName)s primaryID for '%(g)s'\n\n" % locals())
609 dictList.append({
610 "in_ned": 0,
611 "primary_ned_id": thisDict["input_name"]
612 })
614 else:
615 dictList.append({
616 "primary_ned_id": thisDict["input_name"],
617 "in_ned": 0,
618 "primaryID": self.theseIds[thisDict["input_name"]]
619 })
621 self.log.debug(
622 'completed the ``_query_ned_and_add_results_to_database`` method')
623 return dictList
625 def _update_sdss_coverage(
626 self):
627 """ update sdss coverage
629 .. todo ::
631 - update key arguments values and definitions with defaults
632 - update return values and definitions
633 - update usage examples and text
634 - update docstring text
635 - check sublime snippet exists
636 - clip any useful text to docs mindmap
637 - regenerate the docs and check redendering of this docstring
638 """
639 self.log.debug('starting the ``_update_sdss_coverage`` method')
641 tableName = self.dbTableName
643 # SELECT THE LOCATIONS NEEDING TO BE CHECKED
644 sqlQuery = u"""
645 select primary_ned_id, primaryID, raDeg, decDeg, sdss_coverage from %(tableName)s where sdss_coverage is null and master_row = 1 and in_ned = 1 order by dist_mpc;
646 """ % locals()
647 rows = readquery(
648 log=self.log,
649 sqlQuery=sqlQuery,
650 dbConn=self.cataloguesDbConn,
651 quiet=False
652 )
654 totalCount = len(rows)
655 count = 0
656 for row in rows:
657 count += 1
658 if count > 1:
659 # Cursor up three lines and clear
660 sys.stdout.write("\x1b[1A\x1b[2K")
661 sys.stdout.write("\x1b[1A\x1b[2K")
662 sys.stdout.write("\x1b[1A\x1b[2K")
664 if count > totalCount:
665 count = totalCount
666 percent = (old_div(float(count), float(totalCount))) * 100.
668 primaryID = row["primaryID"]
669 raDeg = float(row["raDeg"])
670 decDeg = float(row["decDeg"])
671 primary_ned_id = row["primary_ned_id"]
673 # SDSS CAN ONLY ACCEPT 60 QUERIES/MIN
674 time.sleep(1.1)
675 print("%(count)s / %(totalCount)s (%(percent)1.1f%%) NED galaxies checked for SDSS coverage" % locals())
676 print("NED NAME: ", primary_ned_id)
678 # covered = True | False | 999 (i.e. not sure)
679 sdss_coverage = check_coverage(
680 log=self.log,
681 ra=raDeg,
682 dec=decDeg
683 ).get()
685 if sdss_coverage == 999:
686 sdss_coverage_flag = "null"
687 elif sdss_coverage == True:
688 sdss_coverage_flag = 1
689 elif sdss_coverage == False:
690 sdss_coverage_flag = 0
691 else:
692 self.log.error('could not get sdss coverage' % locals())
693 sys.exit(0)
695 # UPDATE THE DATABASE FLAG
696 sqlQuery = u"""
697 update %(tableName)s set sdss_coverage = %(sdss_coverage_flag)s where primaryID = %(primaryID)s
698 """ % locals()
699 writequery(
700 log=self.log,
701 sqlQuery=sqlQuery,
702 dbConn=self.cataloguesDbConn,
703 )
705 self.log.debug('completed the ``_update_sdss_coverage`` method')
706 return None
708 # use the tab-trigger below for new method
709 # xt-class-method