Coverage for sherlock/imports/veron.py: 93%
91 statements
« prev ^ index » next coverage.py v7.2.2, created at 2023-10-10 13:58 +0000
« prev ^ index » next coverage.py v7.2.2, created at 2023-10-10 13:58 +0000
1#!/usr/local/bin/python
2# encoding: utf-8
3"""
4*import veron catalogue into sherlock-catalogues database*
6:Author:
7 David Young
8"""
9from __future__ import print_function
10from builtins import zip
11import sys
12import os
13os.environ['TERM'] = 'vt100'
14import readline
15import glob
16import pickle
17import codecs
18import string
19import re
20from docopt import docopt
21from ._base_importer import _base_importer
23class veron(_base_importer):
24 """
25 *importer object for the* `VERON AGN catalogue <http://cdsarc.u-strasbg.fr/viz-bin/Cat?VII/258>`_
27 **Key Arguments**
29 - ``dbConn`` -- mysql database connection
30 - ``log`` -- logger
31 - ``settings`` -- the settings dictionary
32 - ``pathToDataFIle`` -- path to the veron data file
33 - ``version`` -- version of the veron catalogue
36 **Usage**
38 To import the veron catalogue catalogue, run the following:
41 ```python
42 from sherlock.imports import veron
43 ```
45 catalogue = veron(
46 log=log,
47 settings=settings,
48 pathToDataFile="/path/to/veron.txt",
49 version="1.0",
50 catalogueName="veron"
51 )
52 catalogue.ingest()
54 Whenever downloading a version of the Veron catalogue from Vizier use the following column selection:
56 .. image:: https://i.imgur.com/4k7MJuw.png
57 :width: 800px
58 :alt: Veron column selection in Vizier
60 .. todo ::
62 - abstract this module out into its own stand alone script
63 - add ppmxl to stand-alone import scripts
64 - update key arguments values and definitions with defaults
65 - update return values and definitions
66 - update usage examples and text
67 - update docstring text
68 - check sublime snippet exists
69 - clip any useful text to docs mindmap
70 - regenerate the docs and check redendering of this docstring
71 """
72 # INITIALISATION
74 def ingest(self):
75 """ingest the veron catalogue into the catalogues database
77 See class docstring for usage.
78 """
79 self.log.debug('starting the ``get`` method')
81 dictList = self._create_dictionary_of_veron()
83 tableName = self.dbTableName
84 createStatement = """
85 CREATE TABLE `%(tableName)s` (
86 `primaryId` bigint(20) NOT NULL AUTO_INCREMENT COMMENT 'An internal counter',
87 `B_V` float DEFAULT NULL,
88 `U_B` float DEFAULT NULL,
89 `abs_magnitude` float DEFAULT NULL,
90 `dateCreated` datetime DEFAULT CURRENT_TIMESTAMP,
91 `decDeg` double DEFAULT NULL,
92 `magnitude` float DEFAULT NULL,
93 `raDeg` double DEFAULT NULL,
94 `class` varchar(10) COLLATE utf8_unicode_ci DEFAULT NULL,
95 `name` varchar(100) COLLATE utf8_unicode_ci DEFAULT NULL,
96 `redshift` float DEFAULT NULL,
97 `not_radio` varchar(10) COLLATE utf8_unicode_ci DEFAULT NULL,
98 `magnitude_filter` varchar(10) COLLATE utf8_unicode_ci DEFAULT 'V',
99 `htm16ID` bigint(20) DEFAULT NULL,
100 `redshift_flag` varchar(100) COLLATE utf8_unicode_ci DEFAULT NULL,
101 `spectral_classification` varchar(100) COLLATE utf8_unicode_ci DEFAULT NULL,
102 `dateLastModified` datetime DEFAULT CURRENT_TIMESTAMP,
103 `updated` varchar(45) DEFAULT '0',
104 `htm10ID` bigint(20) DEFAULT NULL,
105 `htm13ID` bigint(20) DEFAULT NULL,
106 PRIMARY KEY (`primaryId`),
107 UNIQUE KEY `radeg_decdeg` (`raDeg`,`decDeg`),
108 KEY `idx_htm16ID` (`htm16ID`),
109 KEY `idx_htm10ID` (`htm10ID`),
110 KEY `idx_htm13ID` (`htm13ID`)
111 ) ENGINE=MyISAM AUTO_INCREMENT=168945 DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci;
112""" % locals()
114 self.add_data_to_database_table(
115 dictList=dictList,
116 createStatement=createStatement
117 )
119 self.log.debug('completed the ``get`` method')
120 return None
122 def _create_dictionary_of_veron(
123 self):
124 """create a list of dictionaries containing all the rows in the veron catalogue
126 **Return**
128 - ``dictList`` - a list of dictionaries containing all the rows in the veron catalogue
131 .. todo ::
133 - update key arguments values and definitions with defaults
134 - update return values and definitions
135 - update usage examples and text
136 - update docstring text
137 - check sublime snippet exists
138 - clip any useful text to docs mindmap
139 - regenerate the docs and check redendering of this docstring
140 """
141 self.log.debug(
142 'starting the ``_create_dictionary_of_veron`` method')
144 dictList = []
145 try:
146 lines = str.split(self.catData, '\n')
147 except:
148 lines = string.split(self.catData, '\n')
150 totalCount = len(lines)
151 count = 0
152 switch = 0
153 for line in lines:
154 if (len(line) == 0 or line[0] in ["#", " "]) and switch == 0:
155 continue
156 else:
157 switch = 1
158 count += 1
159 if count > 1:
160 # Cursor up one line and clear line
161 sys.stdout.write("\x1b[1A\x1b[2K")
162 print("%(count)s / %(totalCount)s veron data added to memory" % locals())
164 if count == 1:
165 theseKeys = []
166 try:
167 someKeys = str.split(line, '|')
168 except:
169 someKeys = string.split(line, '|')
171 for key in someKeys:
172 if key == "_RAJ2000":
173 key = "raDeg"
174 if key == "_DEJ2000":
175 key = "decDeg"
176 if key == "Cl":
177 key = "class"
178 if key == "nR":
179 key = "not_radio"
180 if key == "Name":
181 key = "name"
182 if key == "l_z":
183 key = "redshift_flag"
184 if key == "z":
185 key = "redshift"
186 if key == "Sp":
187 key = "spectral_classification"
188 if key == "n_Vmag":
189 key = "magnitude_filter"
190 if key == "Vmag":
191 key = "magnitude"
192 if key == "B-V":
193 key = "B_V"
194 if key == "U-B":
195 key = "U_B"
196 if key == "Mabs":
197 key = "abs_magnitude"
198 theseKeys.append(key)
199 continue
201 if count in [2, 3]:
202 continue
204 thisDict = {}
205 try:
206 theseValues = str.split(line, '|')
207 except:
208 theseValues = string.split(line, '|')
210 for k, v in zip(theseKeys, theseValues):
211 v = v.strip()
212 if len(v) == 0 or v == "-":
213 v = None
214 thisDict[k] = v
215 dictList.append(thisDict)
217 self.log.debug(
218 'completed the ``_create_dictionary_of_veron`` method')
219 return dictList
221 # use the tab-trigger below for new method
222 # xt-class-method