Coverage for fundamentals/tools.py: 27%

Hot-keys on this page

r m x p toggle line displays

j k next/prev highlighted chunk

0 (zero) top of page

1 (one) first highlighted chunk

1#!/usr/bin/env python

2# encoding: utf-8

3"""

4*Toolset to setup the main function for a cl-util*

6:Author:

7 David Young

8"""

9from __future__ import print_function

10from __future__ import absolute_import

11from builtins import object

12import sys

13import os

14import yaml

15try:

16 yaml.warnings({'YAMLLoadWarning': False})

17except:

18 pass

19from collections import OrderedDict

20import shutil

21from subprocess import Popen, PIPE, STDOUT

22from . import logs as dl

23import time

24from docopt import docopt

25try:

26 from StringIO import StringIO

27except ImportError:

28 from io import StringIO

29from os.path import expanduser

31###################################################################

32# CLASSES #

33###################################################################

36class tools(object):

37 """

38 *common setup methods & attributes of the main function in cl-util*

40 **Key Arguments**

42 - ``dbConn`` -- mysql database connection

43 - ``arguments`` -- the arguments read in from the command-line

44 - ``docString`` -- pass the docstring from the host module so that docopt can work on the usage text to generate the required arguments

45 - ``logLevel`` -- the level of the logger required. Default *DEBUG*. [DEBUG|INFO|WARNING|ERROR|CRITICAL]

46 - ``options_first`` -- options come before commands in CL usage. Default *False*.

47 - ``projectName`` -- the name of the project, used to create a default settings file in ``~/.config/projectName/projectName.yaml``. Default *False*.

48 - ``distributionName`` -- the distribution name if different from the projectName (i.e. if the package is called by anohter name on PyPi). Default *False*

49 - ``tunnel`` -- will setup a ssh tunnel (if the settings are found in the settings file). Default *False*.

50 - ``defaultSettingsFile`` -- if no settings file is passed via the doc-string use the default settings file in ``~/.config/projectName/projectName.yaml`` (don't have to clutter command-line with settings)

53 **Usage**

55 Add this to the ``__main__`` function of your command-line module

57 ```python

58 # setup the command-line util settings

59 from fundamentals import tools

60 su = tools(

61 arguments=arguments,

62 docString=__doc__,

63 logLevel="DEBUG",

64 options_first=False,

65 projectName="myprojectName"

66 )

67 arguments, settings, log, dbConn = su.setup()

68 ```

70 Here is a template settings file content you could use:

72 ```yaml

73 version: 1

74 database settings:

75 db: unit_tests

76 host: localhost

77 user: utuser

78 password: utpass

79 tunnel: true

81 # SSH TUNNEL - if a tunnel is required to connect to the database(s) then add setup here

82 # Note only one tunnel is setup - may need to change this to 2 tunnels in the future if

83 # code, static catalogue database and transient database are all on seperate machines.

84 ssh tunnel:

85 remote user: username

86 remote ip: mydomain.co.uk

87 remote datbase host: mydatabaseName

88 port: 9002

90 logging settings:

91 formatters:

92 file_style:

93 format: '* %(asctime)s - %(name)s - %(levelname)s (%(pathname)s > %(funcName)s > %(lineno)d) - %(message)s '

94 datefmt: '%Y/%m/%d %H:%M:%S'

95 console_style:

96 format: '* %(asctime)s - %(levelname)s: %(pathname)s:%(funcName)s:%(lineno)d > %(message)s'

97 datefmt: '%H:%M:%S'

98 html_style:

99 format: '<div id="row" class="%(levelname)s">%(asctime)s file:%(filename)s method:%(funcName)s line#:%(lineno)d %(pathname)s <div class="right">%(message)s%(levelname)s</div></div>'

100 datefmt: '%Y-%m-%d %H:%M %Ss'

101 handlers:

102 console:

103 class: logging.StreamHandler

104 level: DEBUG

105 formatter: console_style

106 stream: ext://sys.stdout

107 file:

108 class: logging.handlers.GroupWriteRotatingFileHandler

109 level: WARNING

110 formatter: file_style

111

112

113 filename: /Users/Dave/.config/myprojectName/myprojectName.log

114 mode: w+

115 maxBytes: 102400

116 backupCount: 1

117 root:

118 level: WARNING

119 handlers: [file,console]

120 ```

121 """

122 # Initialisation

123

124 def __init__(

125 self,

126 arguments,

127 docString,

128 logLevel="WARNING",

129 options_first=False,

130 projectName=False,

131 distributionName=False,

132 orderedSettings=False,

133 defaultSettingsFile=False

134 ):

135 self.arguments = arguments

136 self.docString = docString

137 self.logLevel = logLevel

138

139 import psutil

140

141 if not distributionName:

142 distributionName = projectName

143

144 version = '0.0.1'

145 try:

146 import pkg_resources

147 version = pkg_resources.get_distribution(distributionName).version

148 except:

149 pass

150

151 ## ACTIONS BASED ON WHICH ARGUMENTS ARE RECIEVED ##

152 # PRINT COMMAND-LINE USAGE IF NO ARGUMENTS PASSED

153 if arguments == None:

154 arguments = docopt(docString, version="v" + version,

155 options_first=options_first)

156 self.arguments = arguments

157

158 # BUILD A STRING FOR THE PROCESS TO MATCH RUNNING PROCESSES AGAINST

159 lockname = "".join(sys.argv)

160

161 # TEST IF THE PROCESS IS ALREADY RUNNING WITH THE SAME ARGUMENTS (e.g.

162 # FROM CRON) - QUIT IF MATCH FOUND

163 for q in psutil.process_iter():

164 try:

165 this = q.cmdline()

166 except:

167 continue

168

169 test = "".join(this[1:])

170 if q.pid != os.getpid() and lockname == test and "--reload" not in test:

171 thisId = q.pid

172 print("This command is already running (see PID %(thisId)s)" % locals())

173 sys.exit(0)

174

175 try:

176 if "tests.test" in arguments["<pathToSettingsFile>"]:

177 del arguments["<pathToSettingsFile>"]

178 except:

179 pass

180

181 if defaultSettingsFile and "settingsFile" not in arguments and "--settings" not in arguments and os.path.exists(os.getenv(

182 "HOME") + "/.config/%(projectName)s/%(projectName)s.yaml" % locals()):

183 arguments["settingsFile"] = settingsFile = os.getenv(

184 "HOME") + "/.config/%(projectName)s/%(projectName)s.yaml" % locals()

185

186 # UNPACK SETTINGS

187 stream = False

188 if "<settingsFile>" in arguments and arguments["<settingsFile>"]:

189 stream = open(arguments["<settingsFile>"], 'r')

190 elif "<pathToSettingsFile>" in arguments and arguments["<pathToSettingsFile>"]:

191 stream = open(arguments["<pathToSettingsFile>"], 'r')

192 elif "--settingsFile" in arguments and arguments["--settingsFile"]:

193 stream = open(arguments["--settingsFile"], 'r')

194 elif "--settings" in arguments and arguments["--settings"]:

195 stream = open(arguments["--settings"], 'r')

196 elif "pathToSettingsFile" in arguments and arguments["pathToSettingsFile"]:

197 stream = open(arguments["pathToSettingsFile"], 'r')

198 elif "settingsFile" in arguments and arguments["settingsFile"]:

199 stream = open(arguments["settingsFile"], 'r')

200 elif ("settingsFile" in arguments and arguments["settingsFile"] == None) or ("<pathToSettingsFile>" in arguments and arguments["<pathToSettingsFile>"] == None) or ("--settings" in arguments and arguments["--settings"] == None) or ("pathToSettingsFile" in arguments and arguments["pathToSettingsFile"] == None):

201

202 if projectName != False:

203 os.getenv("HOME")

204 projectDir = os.getenv(

205 "HOME") + "/.config/%(projectName)s" % locals()

206 exists = os.path.exists(projectDir)

207 if not exists:

208 # Recursively create missing directories

209 if not os.path.exists(projectDir):

210 os.makedirs(projectDir)

211 settingsFile = os.getenv(

212 "HOME") + "/.config/%(projectName)s/%(projectName)s.yaml" % locals()

213 exists = os.path.exists(settingsFile)

214 arguments["settingsFile"] = settingsFile

215

216 if not exists:

217 import codecs

218 writeFile = codecs.open(

219 settingsFile, encoding='utf-8', mode='w')

220

221 import yaml

222 # GET CONTENT OF YAML FILE AND REPLACE ~ WITH HOME DIRECTORY

223 # PATH

224 with open(settingsFile) as f:

225 content = f.read()

226 home = expanduser("~")

227 content = content.replace("~/", home + "/")

228 astream = StringIO(content)

229

230 if orderedSettings:

231 this = ordered_load(astream, yaml.SafeLoader)

232 else:

233 this = yaml.load(astream)

234 if this:

235

236 settings = this

237 arguments["<settingsFile>"] = settingsFile

238 else:

239 import inspect

240 ds = os.getcwd() + "/rubbish.yaml"

241 level = 0

242 exists = False

243 count = 1

244 while not exists and len(ds) and count < 10:

245 count += 1

246 level -= 1

247 exists = os.path.exists(ds)

248 if not exists:

249 ds = "/".join(inspect.stack()

250 [1][1].split("/")[:level]) + "/default_settings.yaml"

251

252 shutil.copyfile(ds, settingsFile)

253 try:

254 shutil.copyfile(ds, settingsFile)

255 import codecs

256 pathToReadFile = settingsFile

257 try:

258 readFile = codecs.open(

259 pathToReadFile, encoding='utf-8', mode='r')

260 thisData = readFile.read()

261 readFile.close()

262 except IOError as e:

263 message = 'could not open the file %s' % (

264 pathToReadFile,)

265 raise IOError(message)

266 thisData = thisData.replace(

267 "/Users/Dave", os.getenv("HOME"))

268

269 pathToWriteFile = pathToReadFile

270 try:

271 writeFile = codecs.open(

272 pathToWriteFile, encoding='utf-8', mode='w')

273 except IOError as e:

274 message = 'could not open the file %s' % (

275 pathToWriteFile,)

276 raise IOError(message)

277 writeFile.write(thisData)

278 writeFile.close()

279 print(

280 "default settings have been added to '%(settingsFile)s'. Tailor these settings before proceeding to run %(projectName)s" % locals())

281 try:

282 cmd = """open %(pathToReadFile)s""" % locals()

283 p = Popen(cmd, stdout=PIPE,

284 stderr=PIPE, shell=True)

285 except:

286 pass

287 try:

288 cmd = """start %(pathToReadFile)s""" % locals()

289 p = Popen(cmd, stdout=PIPE,

290 stderr=PIPE, shell=True)

291 except:

292 pass

293 except:

294 print(

295 "please add settings to file '%(settingsFile)s'" % locals())

296 # return

297 else:

298 pass

299

300 if stream is not False:

301

302 astream = stream.read()

303 home = expanduser("~")

304 astream = astream.replace("~/", home + "/")

305

306 import yaml

307 if orderedSettings:

308 settings = ordered_load(astream, yaml.SafeLoader)

309 else:

310 settings = yaml.load(astream)

311

312 # SETUP LOGGER -- DEFAULT TO CONSOLE LOGGER IF NONE PROVIDED IN

313 # SETTINGS

314 if 'settings' in locals() and "logging settings" in settings:

315 if "settingsFile" in arguments:

316 log = dl.setup_dryx_logging(

317 yaml_file=arguments["settingsFile"]

318 )

319 elif "<settingsFile>" in arguments:

320 log = dl.setup_dryx_logging(

321 yaml_file=arguments["<settingsFile>"]

322 )

323 elif "<pathToSettingsFile>" in arguments:

324 log = dl.setup_dryx_logging(

325 yaml_file=arguments["<pathToSettingsFile>"]

326 )

327 elif "--settingsFile" in arguments:

328 log = dl.setup_dryx_logging(

329 yaml_file=arguments["--settingsFile"]

330 )

331 elif "pathToSettingsFile" in arguments:

332 log = dl.setup_dryx_logging(

333 yaml_file=arguments["pathToSettingsFile"]

334 )

335

336 elif "--settings" in arguments:

337 log = dl.setup_dryx_logging(

338 yaml_file=arguments["--settings"]

339 )

340

341 elif "--logger" not in arguments or arguments["--logger"] is None:

342 log = dl.console_logger(

343 level=self.logLevel

344 )

345

346 self.log = log

347

348 # unpack remaining cl arguments using `exec` to setup the variable names

349 # automatically

350 for arg, val in list(arguments.items()):

351 if arg[0] == "-":

352 varname = arg.replace("-", "") + "Flag"

353 else:

354 varname = arg.replace("<", "").replace(">", "")

355 if varname == "import":

356 varname = "iimport"

357 if isinstance(val, str):

358 val = val.replace("'", "\\'")

359 exec(varname + " = '%s'" % (val,))

360 else:

361 exec(varname + " = %s" % (val,))

362 if arg == "--dbConn":

363 dbConn = val

364

365 # SETUP A DATABASE CONNECTION BASED ON WHAT ARGUMENTS HAVE BEEN PASSED

366 dbConn = False

367 tunnel = False

368 if ("hostFlag" in locals() and "dbNameFlag" in locals() and hostFlag):

369 # SETUP DB CONNECTION

370 dbConn = True

371 host = arguments["--host"]

372 user = arguments["--user"]

373 passwd = arguments["--passwd"]

374 dbName = arguments["--dbName"]

375 elif 'settings' in locals() and "database settings" in settings and "host" in settings["database settings"]:

376 host = settings["database settings"]["host"]

377 user = settings["database settings"]["user"]

378 passwd = settings["database settings"]["password"]

379 dbName = settings["database settings"]["db"]

380 if "tunnel" in settings["database settings"] and settings["database settings"]["tunnel"]:

381 tunnel = True

382 dbConn = True

383 port = False

384 if "port" in settings["database settings"] and settings["database settings"]["port"]:

385 port = int(settings["database settings"]["port"])

386 else:

387 pass

388

389 if not 'settings' in locals():

390 settings = False

391 self.settings = settings

392

393 if tunnel:

394 self._setup_tunnel()

395 self.dbConn = self.remoteDBConn

396 return None

397

398 if dbConn:

399 import pymysql as ms

400 dbConn = ms.connect(

401 host=host,

402 user=user,

403 passwd=passwd,

404 db=dbName,

405 port=port,

406 use_unicode=True,

407 charset='utf8',

408 local_infile=1,

409 client_flag=ms.constants.CLIENT.MULTI_STATEMENTS,

410 connect_timeout=36000,

411 max_allowed_packet=51200000

412 )

413 dbConn.autocommit(True)

414

415 self.dbConn = dbConn

416

417 return None

418

419 def setup(

420 self):

421 """

422 **Summary:**

423 *setup the attributes and return*

424 """

425

426 return self.arguments, self.settings, self.log, self.dbConn

427

428 def _setup_tunnel(

429 self):

430 """

431 *setup ssh tunnel if required*

432 """

433 from subprocess import Popen, PIPE, STDOUT

434 import pymysql as ms

435

436 # SETUP TUNNEL IF REQUIRED

437 if "ssh tunnel" in self.settings:

438 # TEST TUNNEL DOES NOT ALREADY EXIST

439 sshPort = self.settings["ssh tunnel"]["port"]

440 connected = self._checkServer(

441 self.settings["database settings"]["host"], sshPort)

442 if connected:

443 pass

444 else:

445 # GRAB TUNNEL SETTINGS FROM SETTINGS FILE

446 ru = self.settings["ssh tunnel"]["remote user"]

447 rip = self.settings["ssh tunnel"]["remote ip"]

448 rh = self.settings["ssh tunnel"]["remote datbase host"]

449

450 cmd = "ssh -fnN %(ru)s@%(rip)s -L %(sshPort)s:%(rh)s:3306" % locals()

451 p = Popen(cmd, shell=True, close_fds=True)

452 output = p.communicate()[0]

453

454 # TEST CONNECTION - QUIT AFTER SO MANY TRIES

455 connected = False

456 count = 0

457 while not connected:

458 connected = self._checkServer(

459 self.settings["database settings"]["host"], sshPort)

460 time.sleep(1)

461 count += 1

462 if count == 5:

463 self.log.error(

464 'cound not setup tunnel to remote datbase' % locals())

465 sys.exit(0)

466

467 if "tunnel" in self.settings["database settings"] and self.settings["database settings"]["tunnel"]:

468 # TEST TUNNEL DOES NOT ALREADY EXIST

469 sshPort = self.settings["database settings"]["tunnel"]["port"]

470 connected = self._checkServer(

471 self.settings["database settings"]["host"], sshPort)

472 if connected:

473 pass

474 else:

475 # GRAB TUNNEL SETTINGS FROM SETTINGS FILE

476 ru = self.settings["database settings"][

477 "tunnel"]["remote user"]

478 rip = self.settings["database settings"]["tunnel"]["remote ip"]

479 rh = self.settings["database settings"][

480 "tunnel"]["remote datbase host"]

481

482 cmd = "ssh -fnN %(ru)s@%(rip)s -L %(sshPort)s:%(rh)s:3306" % locals()

483 p = Popen(cmd, shell=True, close_fds=True)

484 output = p.communicate()[0]

485

486 # TEST CONNECTION - QUIT AFTER SO MANY TRIES

487 connected = False

488 count = 0

489 while not connected:

490 connected = self._checkServer(

491 self.settings["database settings"]["host"], sshPort)

492 time.sleep(1)

493 count += 1

494 if count == 5:

495 self.log.error(

496 'cound not setup tunnel to remote datbase' % locals())

497 sys.exit(0)

498

499 # SETUP A DATABASE CONNECTION FOR THE remote database

500 host = self.settings["database settings"]["host"]

501 user = self.settings["database settings"]["user"]

502 passwd = self.settings["database settings"]["password"]

503 dbName = self.settings["database settings"]["db"]

504 thisConn = ms.connect(

505 host=host,

506 user=user,

507 passwd=passwd,

508 db=dbName,

509 port=sshPort,

510 use_unicode=True,

511 charset='utf8',

512 local_infile=1,

513 client_flag=ms.constants.CLIENT.MULTI_STATEMENTS,

514 connect_timeout=36000,

515 max_allowed_packet=51200000

516 )

517 thisConn.autocommit(True)

518 self.remoteDBConn = thisConn

519

520 return None

521

522 def _checkServer(self, address, port):

523 """

524 *Check that the TCP Port we've decided to use for tunnelling is available*

525 """

526 # CREATE A TCP SOCKET

527 import socket

528 s = socket.socket()

529

530 try:

531 s.connect((address, port))

532 return True

533 except socket.error as e:

534 self.log.warning(

535 """Connection to `%(address)s` on port `%(port)s` failed - try again: %(e)s""" % locals())

536 return False

537

538 return None

539

540 # use the tab-trigger below for new method

541 # xt-class-method

542

543###################################################################

544# PUBLIC FUNCTIONS #

545###################################################################

546

547

548def ordered_load(stream, Loader=yaml.Loader, object_pairs_hook=OrderedDict):

549 class OrderedLoader(Loader):

550 pass

551

552 def construct_mapping(loader, node):

553 loader.flatten_mapping(node)

554 return object_pairs_hook(loader.construct_pairs(node))

555 OrderedLoader.add_constructor(

556 yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG,

557 construct_mapping)

558 return yaml.load(stream, OrderedLoader)

559

560if __name__ == '__main__':

561 main()

Coverage for fundamentals/tools.py : 27%

282 statements 76 run 206 missing 0 excluded