Coverage for neddy/namesearch.py: 93%

1#!/usr/local/bin/python 

2# encoding: utf-8 

3""" 

4*Perform a NED name-search and return the metadata for the matched sources* 

5 

6:Author: 

7 David Young 

8""" 

9from __future__ import print_function 

10from neddy import _basesearch 

11import os 

12import sys 

13from future import standard_library 

14standard_library.install_aliases() 

15 

16 

17class namesearch(_basesearch): 

18 """ 

19 *Perform a NED name-search and return the metadata for the matched sources* 

20 

21 **Key Arguments** 

22 

23 - ``log`` -- logger 

24 - ``name`` -- name 

25 - ``quiet`` -- don't print to stdout 

26 - ``verbose`` -- return more metadata for matches 

27 - ``searchParams`` -- list of dictionaries to prepend to results 

28 - ``outputFilePath`` -- path to file to output results to 

29 

30 **Usage** 

31 

32 ```python 

33 from neddy import namesearch 

34 search = namesearch( 

35 log=log, 

36 names=objectName, 

37 verbose=True, 

38 outputFilePath="/path/to/output.csv" 

39 ) 

40 results = search.get() 

41 ``` 

42 

43 """ 

44 

45 def __init__( 

46 self, 

47 log, 

48 names, 

49 quiet=False, 

50 verbose=False, 

51 searchParams=False, 

52 outputFilePath=False 

53 ): 

54 self.log = log 

55 log.debug("instantiating a new 'namesearch' object") 

56 self.names = names 

57 self.quiet = quiet 

58 self.verbose = verbose 

59 self.searchParams = searchParams 

60 self.outputFilePath = outputFilePath 

61 

62 # CREATE A LIST IF SINGLE NAME GIVEN 

63 os.environ['TERM'] = 'vt100' 

64 if not isinstance(self.names, list): 

65 self.uplist = [self.names] 

66 else: 

67 self.uplist = self.names 

68 

69 return None 

70 

71 def get(self): 

72 """ 

73 *perform NED name searches and return the results* 

74 

75 **Return** 

76 

77 - ``results`` -- the search results (list of dictionaries) 

78 """ 

79 self.log.debug('starting the ``get`` method') 

80 

81 # SPLIT THE LIST OF NAMES INTO BATCHES 

82 self.theseBatches, self.theseBatchParams = self._split_incoming_queries_into_batches( 

83 sources=self.uplist, 

84 searchParams=self.searchParams 

85 ) 

86 

87 # PERFORM NAME QUERIES AGAINST NED 

88 self._build_api_url_and_download_results() 

89 self.results, self.headers = self._parse_the_ned_list_results() 

90 self._output_results() 

91 

92 self.log.debug('completed the ``get`` method') 

93 return self.results 

94 

95 def _build_api_url_and_download_results( 

96 self): 

97 """ 

98 *build an API URL call for NED to perform batch name queries and download results* 

99 """ 

100 self.log.debug( 

101 'completed the ````_build_api_url_and_download_results`` method') 

102 

103 import urllib.parse 

104 from fundamentals.download import multiobject_download 

105 

106 baseUrl = "https://ned.ipac.caltech.edu/cgi-bin/" 

107 command = "gmd" 

108 urlParameters = { 

109 "delimiter": "bar", 

110 "NO_LINKS": "1", 

111 "nondb": ["row_count", "user_name_msg", "user_objname"], 

112 "crosid": "objname", 

113 "enotes": "objnote", 

114 "position": ["ra,dec", "bhextin", "pretype", "z", "zunc", "zflag"], 

115 "gadata": ["magnit", "sizemaj", "sizemin", "morphol"], 

116 "attdat_CON": ["M", "S", "H", "R", "z"], 

117 "distance_CON": ["mm", "dmpc"], 

118 "attdat": "attned" 

119 } 

120 

121 queryBase = "%(baseUrl)s%(command)s?uplist=" % locals() 

122 queryList = [] 

123 

124 # BUILD THE LIST OF QUERIES 

125 for batch in self.theseBatches: 

126 thisLength = len(batch) 

127 queryUrl = queryBase 

128 # ADD NAMES 

129 for thisIndex, thisName in enumerate(batch): 

130 queryUrl = queryUrl + urllib.parse.quote(thisName) 

131 if thisIndex < thisLength - 1: 

132 queryUrl = queryUrl + "%0D" 

133 # ADD PARAMETERS 

134 for k, v in list(urlParameters.items()): 

135 if isinstance(v, list): 

136 for item in v: 

137 queryUrl = queryUrl + "&" + \ 

138 k + "=" + urllib.parse.quote(item) 

139 else: 

140 queryUrl = queryUrl + "&" + k + "=" + urllib.parse.quote(v) 

141 queryList.append(queryUrl) 

142 

143 # PULL THE RESULT PAGES FROM NED 

144 self.nedResults = multiobject_download( 

145 urlList=queryList, 

146 downloadDirectory="/tmp", 

147 log=self.log, 

148 timeStamp=1, 

149 timeout=3600, 

150 concurrentDownloads=10, 

151 resetFilename=False, 

152 credentials=False, # { 'username' : "...", "password", "..." } 

153 longTime=True, 

154 indexFilenames=True 

155 ) 

156 

157 for thisIndex, r in enumerate(self.nedResults): 

158 if r == None: 

159 thisUrl = queryList[thisIndex] 

160 self.log.error( 

161 'cound not download NED results for URL %(thisUrl)s' % locals()) 

162 sys.exit(0) 

163 

164 self._convert_html_to_csv() 

165 

166 self.log.debug( 

167 'completed the ``_build_api_url_and_download_results`` method') 

168 return None 

169 

170 def _output_results( 

171 self): 

172 """ 

173 *print the NED search results to STDOUT and/or an output file* 

174 """ 

175 self.log.debug('starting the ``_output_results`` method') 

176 

177 from fundamentals.renderer import list_of_dictionaries 

178 

179 if len(self.results) == 0: 

180 content = "No resuls found" 

181 csvContent = "No resuls found" 

182 else: 

183 dataSet = list_of_dictionaries( 

184 log=self.log, 

185 listOfDictionaries=self.results, 

186 ) 

187 content = dataSet.table(filepath=None) 

188 if self.outputFilePath: 

189 csvContent = dataSet.csv(filepath=self.outputFilePath) 

190 

191 if self.quiet == False: 

192 print(content) 

193 

194 self.log.debug('completed the ``_output_results`` method') 

195 return None