Coverage for sherlock/commonutils/update_wiki_pages.py: 11%
220 statements
« prev ^ index » next coverage.py v7.2.2, created at 2023-10-10 13:58 +0000
« prev ^ index » next coverage.py v7.2.2, created at 2023-10-10 13:58 +0000
1#!/usr/local/bin/python
2# encoding: utf-8
3"""
4*Update sherlock's github wiki pages with some useful info regarding the crossmatch database catalogue tables*
6:Author:
7 David Young
8"""
9from __future__ import print_function
10from builtins import str
11from builtins import object
12import sys
13import os
14os.environ['TERM'] = 'vt100'
15import readline
16import glob
17import collections
18import codecs
19from datetime import datetime, date, time
20import pickle
21from docopt import docopt
22from fundamentals.mysql import readquery
24class update_wiki_pages(object):
25 """
26 *Update sherlock's github wiki pages with some useful info regarding the crossmatch database catalogue tables*
28 **Key Arguments**
30 - ``log`` -- logger
31 - ``settings`` -- the settings dictionary
34 **Usage**
36 To trigger an update of sherlock's wiki pages to give an overview of the crossmatch table database tables run the following:
38 ```python
39 from sherlock.commonutils import update_wiki_pages
40 wiki = update_wiki_pages(
41 log=log,
42 settings=settings
43 )
44 wiki.update()
45 ```
48 .. todo ::
50 - create a new script for updating sherlock wiki with the snippet above, remove wiki command from cl-utils and add stand alone scripts to the sherlock repo (cleans up the usage and docs for sherlock)
51 - harvest text from wiki pages and then delete them: https://github.com/thespacedoctor/sherlock/wiki
52 """
53 # INITIALISATION
55 def __init__(
56 self,
57 log,
58 settings=False,
60 ):
61 self.log = log
62 log.debug("instansiating a new 'update_wiki_pages' object")
63 self.settings = settings
65 # INITIAL ACTIONS
66 # SETUP ALL DATABASE CONNECTIONS
67 from sherlock import database
68 db = database(
69 log=self.log,
70 settings=self.settings
71 )
72 dbConns, dbVersions = db.connect()
73 self.transientsDbConn = dbConns["transients"]
74 self.cataloguesDbConn = dbConns["catalogues"]
76 self.basicColumns = [
77 "view_name",
78 "master table",
79 "description",
80 "version_number",
81 "object_types",
82 "object_type",
83 "number_of_rows",
84 "url",
85 "object_type_accuracy",
86 "last_updated",
87 ]
89 return None
91 def update(self):
92 """
93 Update wiki pages
95 See class docstring for usage
96 """
97 self.log.debug('starting the ``update`` method')
99 if "sherlock wiki root" not in self.settings:
100 print("Sherlock wiki settings not found in settings file")
101 return
102 else:
103 from os.path import expanduser
104 home = expanduser("~")
105 self.settings["sherlock wiki root"] = self.settings[
106 "sherlock wiki root"].replace("~", home)
108 staticTableInfo = self._get_table_infos()
109 viewInfo = self._get_view_infos()
110 streamedTableInfo = self._get_stream_view_infos()
111 self._create_md_tables(
112 tableData=staticTableInfo,
113 viewData=viewInfo,
114 streamData=streamedTableInfo
115 )
116 self._write_wiki_pages()
117 self._update_github()
119 self.log.debug('completed the ``update`` method')
120 return
122 def _get_table_infos(
123 self,
124 trimmed=False):
125 """query the sherlock-catalogues database table metadata
126 """
127 self.log.debug('starting the ``_get_table_infos`` method')
129 sqlQuery = u"""
130 SELECT * FROM tcs_helper_catalogue_tables_info where legacy_table = 0 and table_name not like "legacy%%" and table_name not like "%%stream" order by number_of_rows desc;
131 """ % locals()
132 tableInfo = readquery(
133 log=self.log,
134 sqlQuery=sqlQuery,
135 dbConn=self.cataloguesDbConn,
136 quiet=False
137 )
139 if trimmed:
140 cleanTable = []
141 for r in tableInfo:
142 orow = collections.OrderedDict(sorted({}.items()))
143 for c in self.basicColumns:
144 if c in r:
145 orow[c] = r[c]
146 cleanTable.append(orow)
147 tableInfo = cleanTable
149 self.log.debug('completed the ``_get_table_infos`` method')
150 return tableInfo
152 def _get_view_infos(
153 self,
154 trimmed=False):
155 """query the sherlock-catalogues database view metadata
156 """
157 self.log.debug('starting the ``_get_view_infos`` method')
159 sqlQuery = u"""
160 SELECT v.*, t.description as "master table" FROM tcs_helper_catalogue_views_info as v, tcs_helper_catalogue_tables_info AS t where v.legacy_view = 0 and v.view_name not like "legacy%%" and t.id=v.table_id order by number_of_rows desc
161 """ % locals()
162 viewInfo = readquery(
163 log=self.log,
164 sqlQuery=sqlQuery,
165 dbConn=self.cataloguesDbConn,
166 quiet=False
167 )
169 if trimmed:
170 cleanTable = []
171 for r in viewInfo:
172 orow = collections.OrderedDict(sorted({}.items()))
173 for c in self.basicColumns:
174 if c in r:
175 orow[c] = r[c]
176 cleanTable.append(orow)
177 viewInfo = cleanTable
179 self.log.debug('completed the ``_get_view_infos`` method')
180 return viewInfo
182 def _get_stream_view_infos(
183 self,
184 trimmed=False):
185 """query the sherlock-catalogues database streamed data tables' metadata
186 """
187 self.log.debug('starting the ``_get_stream_view_infos`` method')
189 sqlQuery = u"""
190 SELECT * FROM tcs_helper_catalogue_tables_info where legacy_table = 0 and table_name not like "legacy%%" and table_name like "%%stream" order by number_of_rows desc;
191 """ % locals()
192 streamInfo = readquery(
193 log=self.log,
194 sqlQuery=sqlQuery,
195 dbConn=self.cataloguesDbConn,
196 quiet=False
197 )
199 if trimmed:
200 cleanTable = []
201 for r in streamInfo:
202 orow = collections.OrderedDict(sorted({}.items()))
203 for c in self.basicColumns:
204 if c in r:
205 orow[c] = r[c]
206 cleanTable.append(orow)
207 streamInfo = cleanTable
209 self.log.debug('completed the ``_get_stream_view_infos`` method')
210 return streamInfo
212 def _create_md_tables(
213 self,
214 tableData,
215 viewData,
216 streamData
217 ):
218 """generate markdown format tables from the database query results
220 **Key Arguments**
222 - ``tableData`` -- the sherlock-catalogues database table metadata.
223 - ``viewData`` -- the sherlock-catalogues database view metadata.
224 - ``streamData`` -- the sherlock-catalogues database streamed data tables' metadata.
227 **Return**
229 - None
231 """
232 self.log.debug('starting the ``_create_md_tables`` method')
234 header = u"""
235| <sub>Table Name</sub> | <sub>Description</sub> | <sub>Reference</sub> | <sub>Number Rows</sub> | <sub>Vizier</sub> | <sub>NED</sub> | <sub>Objects</sub> | <sub>Weight (1-10)</sub> |
236| :--- | :--- | :--- | :--- | :--- | :--- | :--- | :--- |"""
238 rows = u""
239 for ti in tableData:
240 table_name = ti["table_name"]
241 description = ti["description"]
242 url = ti["url"]
243 number_of_rows = ti["number_of_rows"]
244 reference_url = ti["reference_url"]
245 reference_text = ti["reference_text"]
246 notes = ti["notes"]
247 vizier_link = ti["vizier_link"]
248 in_ned = ti["in_ned"]
249 object_types = ti["object_types"]
250 version_number = ti["version_number"]
251 last_updated = ti["last_updated"]
252 legacy_table = ti["legacy_table"]
253 old_table_name = ti["old_table_name"]
254 weight = ti["object_type_accuracy"]
256 number_of_rows = str(number_of_rows)
257 thisLen = len(number_of_rows)
258 newNumber = ""
259 count = 0
260 while count < thisLen:
261 count += 1
262 newNumber = number_of_rows[-count] + newNumber
263 if count % 3 == 0:
264 newNumber = "," + newNumber
265 if newNumber[0] == ",":
266 newNumber = newNumber[1:]
268 if vizier_link and len(vizier_link) and vizier_link != 0 and vizier_link != "0":
269 vizier_link = u"[✓](%(vizier_link)s)" % locals()
270 else:
271 vizier_link = u""
273 if in_ned:
274 in_ned = u"✓"
275 else:
276 in_ned = u""
278 rows += u"""
279| <sub>%(table_name)s</sub> | <sub>[%(description)s](%(url)s)</sub> | <sub>[%(reference_text)s](%(reference_url)s)</sub> | <sub>%(newNumber)s</sub> | <sub>%(vizier_link)s</sub> | <sub>%(in_ned)s</sub> | <sub>%(object_types)s</sub> | <sub>%(weight)s</sub> |""" % locals()
281 self.mdTables = header + rows
283 header = u"""
284| <sub>View Name</sub> | <sub>Number Rows</sub> | <sub>Object Type</sub> |
285| :--- | :--- | :--- |"""
287 rows = u""
288 for ti in viewData:
289 view_name = ti["view_name"]
290 number_of_rows = ti["number_of_rows"]
291 object_type = ti["object_type"]
293 number_of_rows = str(number_of_rows)
294 thisLen = len(number_of_rows)
295 newNumber = ""
296 count = 0
297 while count < thisLen:
298 count += 1
299 newNumber = number_of_rows[-count] + newNumber
300 if count % 3 == 0:
301 newNumber = "," + newNumber
302 if newNumber[0] == ",":
303 newNumber = newNumber[1:]
305 rows += u"""
306| <sub>%(view_name)s</sub> | <sub>%(newNumber)s</sub> | <sub>%(object_type)s</sub> |""" % locals()
308 self.mdViews = header + rows
310 header = u"""
311| <sub>Table Name</sub> | <sub>Description</sub> | <sub>Reference</sub> | <sub>Number Rows</sub> | <sub>Objects</sub> |
312| :--- | :--- | :--- | :--- | :--- | """
314 rows = u""
315 for ti in streamData:
316 table_name = ti["table_name"]
317 description = ti["description"]
318 url = ti["url"]
319 number_of_rows = ti["number_of_rows"]
320 reference_url = ti["reference_url"]
321 reference_text = ti["reference_text"]
322 notes = ti["notes"]
323 vizier_link = ti["vizier_link"]
324 in_ned = ti["in_ned"]
325 object_types = ti["object_types"]
326 version_number = ti["version_number"]
327 last_updated = ti["last_updated"]
328 legacy_table = ti["legacy_table"]
329 old_table_name = ti["old_table_name"]
331 number_of_rows = str(number_of_rows)
332 thisLen = len(number_of_rows)
333 newNumber = ""
334 count = 0
335 while count < thisLen:
336 count += 1
337 newNumber = number_of_rows[-count] + newNumber
338 if count % 3 == 0:
339 newNumber = "," + newNumber
340 if newNumber[0] == ",":
341 newNumber = newNumber[1:]
343 if len(vizier_link) and vizier_link != 0 and vizier_link != "0":
344 vizier_link = u"[✓](%(vizier_link)s)" % locals()
345 else:
346 vizier_link = u""
348 if in_ned:
349 in_ned = u"✓"
350 else:
351 in_ned = u""
353 rows += u"""
354| <sub>%(table_name)s</sub> | <sub>[%(description)s](%(url)s)</sub> | <sub>[%(reference_text)s](%(reference_url)s)</sub> | <sub>%(newNumber)s</sub> | <sub>%(object_types)s</sub> |""" % locals()
356 self.mdStreams = header + rows
358 self.log.debug('completed the ``_create_md_tables`` method')
359 return
361 def _write_wiki_pages(
362 self):
363 """write the markdown formated content of the database tables' metadata to local wiki pages
364 """
365 self.log.debug('starting the ``_write_wiki_pages`` method')
367 pathToWriteFile = self.settings[
368 "sherlock wiki root"] + "/Crossmatch-Catalogue Tables.md"
369 writeFile = codecs.open(pathToWriteFile, encoding='utf-8', mode='w')
370 now = datetime.now()
371 now = now.strftime("%Y-%m-%d %H:%M")
372 lastUpdated = """Last Updated %(now)s
373""" % locals()
375 writeFile.write(lastUpdated + self.mdTables)
376 writeFile.close()
378 pathToWriteFile = self.settings[
379 "sherlock wiki root"] + "/Crossmatch-Catalogue Views.md"
380 writeFile = codecs.open(pathToWriteFile, encoding='utf-8', mode='w')
381 now = datetime.now()
382 now = now.strftime("%Y-%m-%d %H:%M")
383 lastUpdated = """Last Updated %(now)s
384""" % locals()
386 writeFile.write(lastUpdated + self.mdViews)
387 writeFile.close()
389 pathToWriteFile = self.settings[
390 "sherlock wiki root"] + "/Crossmatch-Catalogue Streams.md"
391 writeFile = codecs.open(pathToWriteFile, encoding='utf-8', mode='w')
392 now = datetime.now()
393 now = now.strftime("%Y-%m-%d %H:%M")
394 lastUpdated = """Last Updated %(now)s
395""" % locals()
397 writeFile.write(lastUpdated + self.mdStreams)
398 writeFile.close()
400 self.log.debug('completed the ``_write_wiki_pages`` method')
401 return None
403 def _update_github(
404 self):
405 """commit the changes and push them to github
406 """
407 self.log.debug('starting the ``_update_github`` method')
409 from subprocess import Popen, PIPE, STDOUT
410 gdir = self.settings["sherlock wiki root"]
411 cmd = """cd %(gdir)s && git add --all && git commit -m "x" && git pull origin master && git push origin master""" % locals()
412 p = Popen(cmd, stdout=PIPE, stdin=PIPE, shell=True)
413 output = p.communicate()[0]
414 print(output)
415 self.log.debug('output: %(output)s' % locals())
417 self.log.debug('completed the ``_update_github`` method')
418 return None
420 # use the tab-trigger below for new method
421 # xt-class-method