1#!/usr/local/bin/python 

2# encoding: utf-8 

3""" 

4*import the tns stream into the marshall* 

5 

6:Author: 

7 David Young 

8""" 

9from builtins import zip 

10import sys 

11import os 

12os.environ['TERM'] = 'vt100' 

13from fundamentals import tools 

14from ..data import data as basedata 

15from astrocalc.times import now 

16from transientNamer import search 

17from fundamentals.mysql import insert_list_of_dictionaries_into_database_tables 

18 

19 

20class data(basedata): 

21 """ 

22 *Import the tns transient data into the marshall database* 

23 

24 **Key Arguments** 

25 

26 - ``log`` -- logger 

27 - ``dbConn`` -- the marshall database connection 

28 - ``settings`` -- the settings dictionary 

29 

30 

31 **Usage** 

32 

33 To setup your logger, settings and database connections, please use the ``fundamentals`` package (`see tutorial here <http://fundamentals.readthedocs.io/en/latest/#tutorial>`_).  

34 

35 To initiate a data object, use the following: 

36 

37 ```python 

38 from marshallEngine.feeders.tns.data import data 

39 ingester = data( 

40 log=log, 

41 settings=settings, 

42 dbConn=dbConn 

43 ).ingest()  

44 ``` 

45 

46 """ 

47 

48 def __init__( 

49 self, 

50 log, 

51 dbConn, 

52 settings=False, 

53 ): 

54 self.log = log 

55 log.debug("instansiating a new 'data' object") 

56 self.settings = settings 

57 self.dbConn = dbConn 

58 self.fsTableName = "tns_sources" 

59 self.survey = "tns" 

60 

61 # xt-self-arg-tmpx 

62 

63 return None 

64 

65 def ingest( 

66 self, 

67 withinLastDays=False): 

68 """*Ingest the data into the marshall feeder survey table* 

69 

70 **Key Arguments** 

71 

72 - ``withinLastDays`` -- note this will be handle by the transientNamer import to the database 

73 

74 """ 

75 self.log.debug('starting the ``ingest`` method') 

76 

77 # UPDATE THE TNS SPECTRA TABLE WITH EXTRA INFOS 

78 from fundamentals.mysql import writequery 

79 sqlQuery = """CALL `update_tns_tables`();""" % locals() 

80 writequery( 

81 log=self.log, 

82 sqlQuery=sqlQuery, 

83 dbConn=self.dbConn 

84 ) 

85 

86 # PARSE TNS 

87 tns = search( 

88 log=self.log, 

89 discInLastDays=withinLastDays, 

90 settings=self.settings 

91 ) 

92 

93 lists = [tns.sources, tns.photometry, tns.files, tns.spectra] 

94 tableNames = ["tns_sources", "tns_photometry", 

95 "tns_files", "tns_spectra"] 

96 

97 for l, t in zip(lists, tableNames): 

98 # USE dbSettings TO ACTIVATE MULTIPROCESSING - INSERT LIST OF 

99 # DICTIONARIES INTO DATABASE 

100 insert_list_of_dictionaries_into_database_tables( 

101 dbConn=self.dbConn, 

102 log=self.log, 

103 dictList=l, 

104 dbTableName=t, 

105 dateModified=True, 

106 dateCreated=True, 

107 batchSize=2500, 

108 replace=True, 

109 dbSettings=self.settings["database settings"] 

110 ) 

111 

112 # INSERT THE SOURCES TABLE 

113 self.insert_into_transientBucket() 

114 

115 # NOW THE SPECTRA TABLE 

116 self.fsTableName = "tns_spectra" 

117 self.survey = "tns" 

118 self.insert_into_transientBucket(importUnmatched=False) 

119 

120 # NOW THE PHOTOMETRY TABLE 

121 self.fsTableName = "tns_photometry" 

122 self.survey = "tns" 

123 self.insert_into_transientBucket(importUnmatched=False) 

124 

125 # ALSO MATCH NEW ASTRONOTES 

126 sqlQuery = """CALL sync_marshall_feeder_survey_transientBucketId('astronotes_transients');""" % locals( 

127 ) 

128 writequery( 

129 log=self.log, 

130 sqlQuery=sqlQuery, 

131 dbConn=self.dbConn 

132 ) 

133 

134 # CLEAN UP TASKS TO MAKE THE TICKET UPDATE 

135 self.clean_up() 

136 

137 self.log.debug('completed the ``ingest`` method') 

138 return None