1#!/usr/local/bin/python 

2# encoding: utf-8 

3""" 

4*import the useradded stream into the marshall* 

5 

6:Author: 

7 David Young 

8""" 

9from __future__ import print_function 

10import sys 

11import os 

12os.environ['TERM'] = 'vt100' 

13from fundamentals import tools 

14from ..data import data as basedata 

15from astrocalc.times import now 

16from fundamentals.mysql import readquery 

17from marshallEngine.housekeeping import update_transient_summaries 

18 

19 

20class data(basedata): 

21 """ 

22 *Import the useradded transient data into the marshall database* 

23 

24 **Key Arguments** 

25 

26 - ``log`` -- logger 

27 - ``dbConn`` -- the marshall database connection 

28 - ``settings`` -- the settings dictionary 

29 

30 

31 **Usage** 

32 

33 To setup your logger, settings and database connections, please use the ``fundamentals`` package (`see tutorial here <http://fundamentals.readthedocs.io/en/latest/#tutorial>`_). 

34 

35 To initiate a data object, use the following: 

36 

37 ```python 

38 from marshallEngine.feeders.useradded.data import data 

39 ingester = data( 

40 log=log, 

41 settings=settings, 

42 dbConn=dbConn 

43 ).ingest(withinLastDays=withInLastDay) 

44 ``` 

45 

46 """ 

47 # Initialisation 

48 

49 def __init__( 

50 self, 

51 log, 

52 dbConn, 

53 settings=False, 

54 ): 

55 self.log = log 

56 log.debug("instansiating a new 'data' object") 

57 self.settings = settings 

58 self.dbConn = dbConn 

59 

60 self.fsTableName = "fs_user_added" 

61 self.survey = "useradded" 

62 

63 # xt-self-arg-tmpx 

64 

65 return None 

66 

67 def ingest( 

68 self, 

69 withinLastDays): 

70 """*Ingest the data into the marshall feeder survey table* 

71 

72 **Key Arguments** 

73 

74 - ``withinLastDays`` -- within the last number of days. *Default: 50* 

75 

76 """ 

77 self.log.info('starting the ``ingest`` method') 

78 

79 allLists = [] 

80 

81 self.dictList = allLists 

82 self._import_to_feeder_survey_table() 

83 

84 self.insert_into_transientBucket( 

85 updateTransientSummaries=False) 

86 

87 sqlQuery = u""" 

88 select transientBucketId from fs_user_added where transientBucketId is not null order by dateCreated desc limit 1 

89 """ % locals() 

90 rows = readquery( 

91 log=self.log, 

92 sqlQuery=sqlQuery, 

93 dbConn=self.dbConn 

94 ) 

95 

96 if len(rows): 

97 transientBucketId = rows[0]["transientBucketId"] 

98 print(transientBucketId) 

99 else: 

100 transientBucketId = False 

101 

102 # UPDATE THE TRANSIENT BUCKET SUMMARY TABLE IN THE MARSHALL DATABASE 

103 updater = update_transient_summaries( 

104 log=self.log, 

105 settings=self.settings, 

106 dbConn=self.dbConn, 

107 transientBucketId=transientBucketId 

108 ) 

109 updater.update() 

110 

111 # CLEAN UP TASKS TO MAKE THE TICKET UPDATE 

112 self.clean_up() 

113 

114 self.log.info('completed the ``ingest`` method') 

115 return None 

116 

117 def _clean_data_pre_ingest( 

118 self, 

119 surveyName, 

120 withinLastDays=False): 

121 """*clean up the list of dictionaries containing the useradded data, pre-ingest* 

122 

123 **Key Arguments** 

124 

125 - ``surveyName`` -- the useradded survey name 

126 - ``withinLastDays`` -- the lower limit of observations to include (within the last N days from now). Default *False*, i.e. no limit 

127 

128 

129 **Return** 

130 

131 - ``dictList`` -- the cleaned list of dictionaries ready for ingest 

132 

133 

134 **Usage** 

135 

136 To clean the data from the useradded survey: 

137 

138 ```python 

139 dictList = ingesters._clean_data_pre_ingest(surveyName="useradded") 

140 ``` 

141 

142 Note you will also be able to access the data via ``ingester.dictList`` 

143 

144 """ 

145 self.log.info('starting the ``_clean_data_pre_ingest`` method') 

146 

147 self.dictList = [] 

148 

149 # CALC MJD LIMIT 

150 if withinLastDays: 

151 mjdLimit = now( 

152 log=self.log 

153 ).get_mjd() - float(withinLastDays) 

154 

155 for row in self.csvDicts: 

156 # IF NOW IN THE LAST N DAYS - SKIP 

157 if withinLastDays and float(row["mjd_obs"]) < mjdLimit: 

158 continue 

159 

160 # MASSAGE THE DATA IN THE INPT FORMAT TO WHAT IS NEEDED IN THE 

161 # FEEDER SURVEY TABLE IN THE DATABASE 

162 thisDictionary = {} 

163 # thisDictionary["candidateID"] = row["ps1_designation"] 

164 # ... 

165 

166 self.dictList.append(thisDictionary) 

167 

168 self.log.info('completed the ``_clean_data_pre_ingest`` method') 

169 return self.dictList 

170 

171 # use the tab-trigger below for new method 

172 # xt-class-method