1#!/usr/bin/env python 

2# encoding: utf-8 

3""" 

4*import the atels stream into the marshall* 

5 

6:Author: 

7 David Young 

8 

9:Date Created: 

10 January 12, 2021 

11""" 

12################# GLOBAL IMPORTS #################### 

13import sys 

14import os 

15os.environ['TERM'] = 'vt100' 

16from fundamentals import tools 

17from ..data import data as basedata 

18from astrocalc.times import now 

19import git 

20from fundamentals.mysql import writequery 

21 

22 

23class data(basedata): 

24 """ 

25 *Import the atels transient data into the marshall database* 

26 

27 **Key Arguments:** 

28 - ``log`` -- logger 

29 - ``dbConn`` -- the marshall database connection 

30 - ``settings`` -- the settings dictionary 

31 

32 **Usage:** 

33 

34 To setup your logger, settings and database connections, please use the ``fundamentals`` package (`see tutorial here <http://fundamentals.readthedocs.io/en/latest/#tutorial>`_).  

35 

36 To initiate a data object, use the following: 

37 

38 .. code-block:: python  

39 

40 from marshallEngine.feeders.atels.data import data 

41 ingester = data( 

42 log=log, 

43 settings=settings, 

44 dbConn=dbConn 

45 ).ingest(withinLastDays=withInLastDay)  

46 """ 

47 # Initialisation 

48 

49 def __init__( 

50 self, 

51 log, 

52 dbConn, 

53 settings=False, 

54 ): 

55 self.log = log 

56 log.debug("instansiating a new 'data' object") 

57 self.settings = settings 

58 self.dbConn = dbConn 

59 

60 self.fsTableName = "atel_coordinates" 

61 self.survey = "atel" 

62 self.dictList = [] 

63 

64 # xt-self-arg-tmpx 

65 

66 return None 

67 

68 def ingest( 

69 self, 

70 withinLastDays): 

71 """*Ingest the data into the marshall feeder survey table* 

72 

73 **Key Arguments:** 

74 - ``withinLastDays`` -- within the last number of days. *Default: 50* 

75 """ 

76 self.log.debug('starting the ``ingest`` method') 

77 

78 allLists = [] 

79 

80 self.download_new_atels() 

81 self.parse_atels_to_database() 

82 

83 self.insert_into_transientBucket( 

84 importUnmatched=False, 

85 updateTransientSummaries=True) 

86 

87 self.fsTableName = "atel_names" 

88 self.insert_into_transientBucket( 

89 importUnmatched=False, 

90 updateTransientSummaries=True) 

91 

92 sqlQuery = f"""CALL insert_atel_titles_to_comments();""" 

93 writequery( 

94 log=self.log, 

95 sqlQuery=sqlQuery, 

96 dbConn=self.dbConn 

97 ) 

98 

99 # CLEAN UP TASKS TO MAKE THE TICKET UPDATE 

100 self.clean_up() 

101 

102 self.log.debug('completed the ``ingest`` method') 

103 return None 

104 

105 def update_git_repo( 

106 self): 

107 """*update the atel data git repo (if it exists)* 

108 ``` 

109 """ 

110 self.log.debug('starting the ``update_git_repo`` method') 

111 

112 # TEST IF atel-directory IS A GIT REPO 

113 atelDir = self.settings["atel-directory"] 

114 

115 # FIRST TRY RAW PATH 

116 try: 

117 repo = git.Repo(atelDir) 

118 except: 

119 repo = False 

120 

121 # IF THIS FAILS TRY PARENT DIRECTORY 

122 if repo == False: 

123 try: 

124 repo = os.path.dirname(atelDir) 

125 repo = git.Repo(repo) 

126 except: 

127 repo = False 

128 

129 # PUSH/PULL CHAGES 

130 if repo != False: 

131 repo.git.add(update=True) 

132 repo.index.commit("adding atels") 

133 o = repo.remotes.origin 

134 o.pull() 

135 o.push() 

136 o.pull() 

137 

138 self.log.debug('completed the ``update_git_repo`` method') 

139 return None 

140 

141 def download_new_atels( 

142 self): 

143 """*download new atel html files* 

144 """ 

145 self.log.debug('starting the ``download_new_atels`` method') 

146 

147 self.update_git_repo() 

148 from atelParser import download 

149 atels = download( 

150 log=self.log, 

151 settings=self.settings 

152 ) 

153 atelsToDownload = atels.get_list_of_atels_still_to_download() 

154 if "test" in self.settings and self.settings["test"] == True: 

155 atelsToDownload = [atelsToDownload[-1]] 

156 atels.maxsleep = 1 

157 else: 

158 atels.maxsleep = 20 

159 atels.download_list_of_atels(atelsToDownload) 

160 self.update_git_repo() 

161 

162 self.log.debug('completed the ``download_new_atels`` method') 

163 return None 

164 

165 def parse_atels_to_database( 

166 self): 

167 """*parse content of atels to the marshall database.* 

168 

169 This populates the `atel_coordinates`, `atel_fullcontent` and `atel_names` database tables. 

170 """ 

171 self.log.debug('starting the ``parse_atels_to_database`` method') 

172 

173 from atelParser import mysql 

174 parser = mysql( 

175 log=self.log, 

176 settings=self.settings 

177 ) 

178 parser.atels_to_database() 

179 parser.parse_atels() 

180 parser.populate_htm_columns() 

181 

182 self.log.debug('completed the ``parse_atels_to_database`` method') 

183 return None 

184 

185 # use the tab-trigger below for new method 

186 # xt-class-method