Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1#!/usr/local/bin/python 

2# encoding: utf-8 

3""" 

4*import veron catalogue into sherlock-catalogues database* 

5 

6:Author: 

7 David Young 

8""" 

9from __future__ import print_function 

10from builtins import zip 

11import sys 

12import os 

13os.environ['TERM'] = 'vt100' 

14import readline 

15import glob 

16import pickle 

17import codecs 

18import string 

19import re 

20from docopt import docopt 

21from ._base_importer import _base_importer 

22 

23class veron(_base_importer): 

24 """ 

25 *importer object for the* `VERON AGN catalogue <http://cdsarc.u-strasbg.fr/viz-bin/Cat?VII/258>`_ 

26 

27 **Key Arguments** 

28 

29 - ``dbConn`` -- mysql database connection 

30 - ``log`` -- logger 

31 - ``settings`` -- the settings dictionary 

32 - ``pathToDataFIle`` -- path to the veron data file 

33 - ``version`` -- version of the veron catalogue 

34  

35 

36 **Usage** 

37 

38 To import the veron catalogue catalogue, run the following: 

39  

40 

41 ```python 

42 from sherlock.imports import veron 

43 ``` 

44 

45 catalogue = veron( 

46 log=log, 

47 settings=settings, 

48 pathToDataFile="/path/to/veron.txt", 

49 version="1.0", 

50 catalogueName="veron" 

51 ) 

52 catalogue.ingest() 

53 

54 Whenever downloading a version of the Veron catalogue from Vizier use the following column selection: 

55 

56 .. image:: https://i.imgur.com/4k7MJuw.png 

57 :width: 800px 

58 :alt: Veron column selection in Vizier 

59 

60 .. todo :: 

61 

62 - abstract this module out into its own stand alone script 

63 - add ppmxl to stand-alone import scripts 

64 - update key arguments values and definitions with defaults 

65 - update return values and definitions 

66 - update usage examples and text 

67 - update docstring text 

68 - check sublime snippet exists 

69 - clip any useful text to docs mindmap 

70 - regenerate the docs and check redendering of this docstring 

71 """ 

72 # INITIALISATION 

73 

74 def ingest(self): 

75 """ingest the veron catalogue into the catalogues database 

76 

77 See class docstring for usage. 

78 """ 

79 self.log.debug('starting the ``get`` method') 

80 

81 dictList = self._create_dictionary_of_veron() 

82 

83 tableName = self.dbTableName 

84 createStatement = """ 

85 CREATE TABLE `%(tableName)s` ( 

86 `primaryId` bigint(20) NOT NULL AUTO_INCREMENT COMMENT 'An internal counter', 

87 `B_V` float DEFAULT NULL, 

88 `U_B` float DEFAULT NULL, 

89 `abs_magnitude` float DEFAULT NULL, 

90 `dateCreated` datetime DEFAULT CURRENT_TIMESTAMP, 

91 `decDeg` double DEFAULT NULL, 

92 `magnitude` float DEFAULT NULL, 

93 `raDeg` double DEFAULT NULL, 

94 `class` varchar(10) COLLATE utf8_unicode_ci DEFAULT NULL, 

95 `name` varchar(100) COLLATE utf8_unicode_ci DEFAULT NULL, 

96 `redshift` float DEFAULT NULL, 

97 `not_radio` varchar(10) COLLATE utf8_unicode_ci DEFAULT NULL, 

98 `magnitude_filter` varchar(10) COLLATE utf8_unicode_ci DEFAULT 'V', 

99 `htm16ID` bigint(20) DEFAULT NULL, 

100 `redshift_flag` varchar(100) COLLATE utf8_unicode_ci DEFAULT NULL, 

101 `spectral_classification` varchar(100) COLLATE utf8_unicode_ci DEFAULT NULL, 

102 `dateLastModified` datetime DEFAULT CURRENT_TIMESTAMP, 

103 `updated` varchar(45) DEFAULT '0', 

104 `htm10ID` bigint(20) DEFAULT NULL, 

105 `htm13ID` bigint(20) DEFAULT NULL, 

106 PRIMARY KEY (`primaryId`), 

107 UNIQUE KEY `radeg_decdeg` (`raDeg`,`decDeg`), 

108 KEY `idx_htm16ID` (`htm16ID`), 

109 KEY `idx_htm10ID` (`htm10ID`), 

110 KEY `idx_htm13ID` (`htm13ID`) 

111 ) ENGINE=MyISAM AUTO_INCREMENT=168945 DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci; 

112""" % locals() 

113 

114 self.add_data_to_database_table( 

115 dictList=dictList, 

116 createStatement=createStatement 

117 ) 

118 

119 self.log.debug('completed the ``get`` method') 

120 return None 

121 

122 def _create_dictionary_of_veron( 

123 self): 

124 """create a list of dictionaries containing all the rows in the veron catalogue 

125 

126 **Return** 

127 

128 - ``dictList`` - a list of dictionaries containing all the rows in the veron catalogue 

129  

130 

131 .. todo :: 

132 

133 - update key arguments values and definitions with defaults 

134 - update return values and definitions 

135 - update usage examples and text 

136 - update docstring text 

137 - check sublime snippet exists 

138 - clip any useful text to docs mindmap 

139 - regenerate the docs and check redendering of this docstring 

140 """ 

141 self.log.debug( 

142 'starting the ``_create_dictionary_of_veron`` method') 

143 

144 dictList = [] 

145 try: 

146 lines = str.split(self.catData, '\n') 

147 except: 

148 lines = string.split(self.catData, '\n') 

149 

150 totalCount = len(lines) 

151 count = 0 

152 switch = 0 

153 for line in lines: 

154 if (len(line) == 0 or line[0] in ["#", " "]) and switch == 0: 

155 continue 

156 else: 

157 switch = 1 

158 count += 1 

159 if count > 1: 

160 # Cursor up one line and clear line 

161 sys.stdout.write("\x1b[1A\x1b[2K") 

162 print("%(count)s / %(totalCount)s veron data added to memory" % locals()) 

163 

164 if count == 1: 

165 theseKeys = [] 

166 try: 

167 someKeys = str.split(line, '|') 

168 except: 

169 someKeys = string.split(line, '|') 

170 

171 for key in someKeys: 

172 if key == "_RAJ2000": 

173 key = "raDeg" 

174 if key == "_DEJ2000": 

175 key = "decDeg" 

176 if key == "Cl": 

177 key = "class" 

178 if key == "nR": 

179 key = "not_radio" 

180 if key == "Name": 

181 key = "name" 

182 if key == "l_z": 

183 key = "redshift_flag" 

184 if key == "z": 

185 key = "redshift" 

186 if key == "Sp": 

187 key = "spectral_classification" 

188 if key == "n_Vmag": 

189 key = "magnitude_filter" 

190 if key == "Vmag": 

191 key = "magnitude" 

192 if key == "B-V": 

193 key = "B_V" 

194 if key == "U-B": 

195 key = "U_B" 

196 if key == "Mabs": 

197 key = "abs_magnitude" 

198 theseKeys.append(key) 

199 continue 

200 

201 if count in [2, 3]: 

202 continue 

203 

204 thisDict = {} 

205 try: 

206 theseValues = str.split(line, '|') 

207 except: 

208 theseValues = string.split(line, '|') 

209 

210 for k, v in zip(theseKeys, theseValues): 

211 v = v.strip() 

212 if len(v) == 0 or v == "-": 

213 v = None 

214 thisDict[k] = v 

215 dictList.append(thisDict) 

216 

217 self.log.debug( 

218 'completed the ``_create_dictionary_of_veron`` method') 

219 return dictList 

220 

221 # use the tab-trigger below for new method 

222 # xt-class-method