1#!/usr/local/bin/python 

2# encoding: utf-8 

3""" 

4*Import Multi Unit Spectroscopic Explorer (MUSE) IFS galaxy stream into sherlock-catalogues database* 

5 

6:Author: 

7 David Young 

8""" 

9from __future__ import print_function 

10import sys 

11import os 

12os.environ['TERM'] = 'vt100' 

13import readline 

14import glob 

15import pickle 

16import codecs 

17import string 

18import requests 

19import re 

20from docopt import docopt 

21from astrocalc.coords import unit_conversion 

22from fundamentals.download import multiobject_download 

23from ._base_importer import _base_importer 

24 

25class ifs(_base_importer): 

26 """ 

27 *Importer for the Multi Unit Spectroscopic Explorer (MUSE) IFS galaxy catalogue stream* 

28 

29 **Key Arguments** 

30 

31 - ``log`` -- logger 

32 - ``settings`` -- the settings dictionary 

33  

34 

35 **Usage** 

36 

37 To import the IFS catalogue stream into the sherlock-catalogues database, run the following: 

38  

39 

40 ```python 

41 from sherlock.imports import IFS 

42 ``` 

43 

44 stream = IFS( 

45 log=log, 

46 settings=settings 

47 ) 

48 stream.ingest() 

49 

50 .. todo :: 

51 

52 - abstract this module out into its own stand alone script 

53 - check sublime snippet exists 

54 """ 

55 # INITIALISATION 

56 

57 def ingest(self): 

58 """*Import the IFS catalogue into the sherlock-catalogues database* 

59 

60 The method first generates a list of python dictionaries from the IFS datafile, imports this list of dictionaries into a database table and then generates the HTMIDs for that table.  

61 

62 **Usage** 

63 

64 See class docstring for usage 

65  

66 """ 

67 self.log.debug('starting the ``get`` method') 

68 

69 self.primaryIdColumnName = "primaryId" 

70 self.raColName = "raDeg" 

71 self.declColName = "decDeg" 

72 self.dbTableName = "tcs_cat_ifs_stream" 

73 self.databaseInsertbatchSize = 500 

74 

75 dictList = self._create_dictionary_of_IFS() 

76 

77 tableName = self.dbTableName 

78 createStatement = """ 

79 CREATE TABLE `%(tableName)s` ( 

80 `primaryId` bigint(20) NOT NULL AUTO_INCREMENT COMMENT 'An internal counter', 

81 `dateCreated` datetime DEFAULT CURRENT_TIMESTAMP, 

82 `decDeg` double DEFAULT NULL, 

83 `name` varchar(100) COLLATE utf8_unicode_ci DEFAULT NULL, 

84 `raDeg` double DEFAULT NULL, 

85 `z` double DEFAULT NULL, 

86 `htm16ID` bigint(20) DEFAULT NULL, 

87 `htm10ID` bigint(20) DEFAULT NULL, 

88 `htm13ID` bigint(20) DEFAULT NULL, 

89 `dateLastModified` datetime DEFAULT CURRENT_TIMESTAMP, 

90 `updated` varchar(45) DEFAULT '0', 

91 PRIMARY KEY (`primaryId`), 

92 UNIQUE KEY `radeg_decdeg` (`raDeg`,`decDeg`), 

93 KEY `idx_htm16ID` (`htm16ID`), 

94 KEY `idx_htm10ID` (`htm10ID`), 

95 KEY `idx_htm13ID` (`htm13ID`) 

96 ) ENGINE=MyISAM AUTO_INCREMENT=0 DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci; 

97""" % locals() 

98 

99 self.add_data_to_database_table( 

100 dictList=dictList, 

101 createStatement=createStatement 

102 ) 

103 

104 self.log.debug('completed the ``get`` method') 

105 return None 

106 

107 def _create_dictionary_of_IFS( 

108 self): 

109 """*Generate the list of dictionaries containing all the rows in the IFS stream* 

110 

111 **Return** 

112 

113 - ``dictList`` - a list of dictionaries containing all the rows in the IFS stream 

114  

115 

116 **Usage** 

117 

118 ```python 

119 from sherlock.imports import IFS 

120 stream = IFS( 

121 log=log, 

122 settings=settings 

123 ) 

124 dictList = stream._create_dictionary_of_IFS() 

125 ``` 

126  

127 """ 

128 self.log.debug( 

129 'starting the ``_create_dictionary_of_IFS`` method') 

130 

131 # GRAB THE CONTENT OF THE IFS CSV 

132 try: 

133 response = requests.get( 

134 url=self.settings["ifs galaxies url"], 

135 ) 

136 thisData = response.content 

137 thisData = str(thisData).split("\n") 

138 status_code = response.status_code 

139 except requests.exceptions.RequestException: 

140 print('HTTP Request failed') 

141 sys.exit(0) 

142 

143 dictList = [] 

144 columns = ["name", "raDeg", "decDeg", "z"] 

145 

146 for line in thisData: 

147 thisDict = {} 

148 line = line.strip() 

149 line = line.replace("\t", " ") 

150 values = line.split("|") 

151 if len(values) > 3: 

152 thisDict["name"] = values[0].strip() 

153 

154 # ASTROCALC UNIT CONVERTER OBJECT 

155 converter = unit_conversion( 

156 log=self.log 

157 ) 

158 try: 

159 raDeg = converter.ra_sexegesimal_to_decimal( 

160 ra=values[1].strip() 

161 ) 

162 thisDict["raDeg"] = raDeg 

163 decDeg = converter.dec_sexegesimal_to_decimal( 

164 dec=values[2].strip() 

165 ) 

166 thisDict["decDeg"] = decDeg 

167 except: 

168 name = thisDict["name"] 

169 self.log.warning( 

170 'Could not convert the coordinates for IFS source %(name)s. Skipping import of this source.' % locals()) 

171 continue 

172 try: 

173 z = float(values[3].strip()) 

174 if z > 0.: 

175 thisDict["z"] = float(values[3].strip()) 

176 else: 

177 thisDict["z"] = None 

178 except: 

179 thisDict["z"] = None 

180 dictList.append(thisDict) 

181 

182 self.log.debug( 

183 'completed the ``_create_dictionary_of_IFS`` method') 

184 return dictList 

185 

186 # use the tab-trigger below for new method 

187 # xt-class-method