Coverage for Importer.py: 0%
478 statements
« prev ^ index » next coverage.py v7.6.7, created at 2024-11-18 00:10 +0000
1#!/usr/bin/env python
2# -*- coding: utf-8 -*-
4# Copyright 2008-2011 Steffen Schaumburg
5# This program is free software: you can redistribute it and/or modify
6# it under the terms of the GNU Affero General Public License as published by
7# the Free Software Foundation, version 3 of the License.
8#
9# This program is distributed in the hope that it will be useful,
10# but WITHOUT ANY WARRANTY; without even the implied warranty of
11# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12# GNU General Public License for more details.
13#
14# You should have received a copy of the GNU Affero General Public License
15# along with this program. If not, see <http://www.gnu.org/licenses/>.
16# In the "official" distribution you can find the license in agpl-3.0.txt.
18from __future__ import print_function
19from __future__ import division
22from past.utils import old_div
23# import L10n
24# _ = L10n.get_translation()
26# Standard Library modules
28import os # todo: remove this once import_dir is in fpdb_import
29from time import time, process_time
30import datetime
31import shutil
32import re
33import zmq
35import logging
38from PyQt5.QtWidgets import QProgressBar, QLabel, QDialog, QVBoxLayout
39from PyQt5.QtCore import QCoreApplication
41# fpdb/FreePokerTools modules
43import Database
45import Configuration
47import IdentifySite
49from Exceptions import FpdbParseError, FpdbHandDuplicate, FpdbHandPartial
51try:
52 import xlrd
53except ImportError:
54 xlrd = None
56if __name__ == "__main__":
57 Configuration.set_logfile("fpdb-log.txt")
58# logging has been set up in fpdb.py or HUD_main.py, use their settings:
59log = logging.getLogger("importer")
62class ZMQSender:
63 def __init__(self, port="5555"):
64 self.context = zmq.Context()
65 self.socket = self.context.socket(zmq.PUSH)
66 self.socket.bind(f"tcp://127.0.0.1:{port}")
67 log.info(f"ZMQ sender initialized on port {port}")
69 def send_hand_id(self, hand_id):
70 try:
71 self.socket.send_string(str(hand_id))
72 log.debug(f"Sent hand ID {hand_id} via ZMQ")
73 except zmq.ZMQError as e:
74 log.error(f"Failed to send hand ID {hand_id}: {e}")
76 def close(self):
77 self.socket.close()
78 self.context.term()
79 log.info("ZMQ sender closed")
82class Importer(object):
83 def __init__(self, caller, settings, config, sql=None, parent=None):
84 """Constructor"""
85 self.settings = settings
86 self.caller = caller
87 self.config = config
88 self.sql = sql
89 self.parent = parent
91 self.idsite = IdentifySite.IdentifySite(config)
93 self.filelist = {}
94 self.dirlist = {}
95 self.siteIds = {}
96 self.removeFromFileList = {} # to remove deleted files
97 self.monitor = False
98 self.updatedsize = {}
99 self.updatedtime = {}
100 self.lines = None
101 self.faobs = None # File as one big string
102 self.mode = None
103 self.pos_in_file = {} # dict to remember how far we have read in the file
105 # Configuration des paramètres par défaut
106 self.callHud = self.config.get_import_parameters().get("callFpdbHud")
107 self.settings.setdefault("handCount", 0)
108 self.settings.setdefault("writeQSize", 1000)
109 self.settings.setdefault("writeQMaxWait", 10)
110 self.settings.setdefault("dropIndexes", "don't drop")
111 self.settings.setdefault("dropHudCache", "don't drop")
112 self.settings.setdefault("starsArchive", False)
113 self.settings.setdefault("ftpArchive", False)
114 self.settings.setdefault("testData", False)
115 self.settings.setdefault("cacheHHC", False)
117 self.writeq = None
118 self.database = Database.Database(self.config, sql=self.sql)
119 self.writerdbs = []
120 self.settings.setdefault("threads", 1)
121 for i in range(self.settings["threads"]):
122 self.writerdbs.append(Database.Database(self.config, sql=self.sql))
124 # Modification : spécifier le port pour ZMQ
125 self.zmq_sender = None
126 process_time() # init clock in windows
128 # Set functions
129 def setMode(self, value):
130 self.mode = value
132 def setCallHud(self, value):
133 self.callHud = value
135 def setCacheSessions(self, value):
136 self.cacheSessions = value
138 def setHandCount(self, value):
139 self.settings["handCount"] = int(value)
141 def setQuiet(self, value):
142 self.settings["quiet"] = value
144 def setHandsInDB(self, value):
145 self.settings["handsInDB"] = value
147 def setThreads(self, value):
148 self.settings["threads"] = value
149 if self.settings["threads"] > len(self.writerdbs):
150 for i in range(self.settings["threads"] - len(self.writerdbs)):
151 self.writerdbs.append(Database.Database(self.config, sql=self.sql))
153 def setDropIndexes(self, value):
154 self.settings["dropIndexes"] = value
156 def setDropHudCache(self, value):
157 self.settings["dropHudCache"] = value
159 def setStarsArchive(self, value):
160 self.settings["starsArchive"] = value
162 def setFTPArchive(self, value):
163 self.settings["ftpArchive"] = value
165 def setPrintTestData(self, value):
166 self.settings["testData"] = value
168 def setFakeCacheHHC(self, value):
169 self.settings["cacheHHC"] = value
171 def getCachedHHC(self):
172 return self.handhistoryconverter
174 # def setWatchTime(self):
175 # self.updated = time()
177 def clearFileList(self):
178 self.updatedsize = {}
179 self.updatetime = {}
180 self.pos_in_file = {}
181 self.filelist = {}
183 def logImport(self, type, file, stored, dups, partial, skipped, errs, ttime, id):
184 hands = stored + dups + partial + skipped + errs
185 now = datetime.datetime.utcnow()
186 ttime100 = ttime * 100
187 self.database.updateFile([type, now, now, hands, stored, dups, partial, skipped, errs, ttime100, True, id])
188 self.database.commit()
190 def addFileToList(self, fpdbfile):
191 """FPDBFile"""
192 file = os.path.splitext(os.path.basename(fpdbfile.path))[0]
193 try: # TODO: this is a dirty hack. GBI needs it, GAI fails with it.
194 file = str(file, "utf8", "replace")
195 except TypeError:
196 pass
197 fpdbfile.fileId = self.database.get_id(file)
198 if not fpdbfile.fileId:
199 now = datetime.datetime.utcnow()
200 fpdbfile.fileId = self.database.storeFile([file, fpdbfile.site.name, now, now, 0, 0, 0, 0, 0, 0, 0, False])
201 self.database.commit()
203 # Add an individual file to filelist
204 def addImportFile(self, filename, site="auto"):
205 # DEBUG->print("addimportfile: filename is a", filename.__class__, filename)
206 # filename not guaranteed to be unicode
207 if self.filelist.get(filename) is not None or not os.path.exists(filename):
208 return False
210 self.idsite.processFile(filename)
211 if self.idsite.get_fobj(filename):
212 fpdbfile = self.idsite.filelist[filename]
213 else:
214 log.error("Importer.addImportFile: siteId Failed for: '%s'" % filename)
215 return False
217 self.addFileToList(fpdbfile)
218 self.filelist[filename] = fpdbfile
219 if site not in self.siteIds:
220 # Get id from Sites table in DB
221 result = self.database.get_site_id(fpdbfile.site.name)
222 if len(result) == 1:
223 self.siteIds[fpdbfile.site.name] = result[0][0]
224 else:
225 if len(result) == 0:
226 log.error(("Database ID for %s not found") % fpdbfile.site.name)
227 else:
228 log.error(("More than 1 Database ID found for %s") % fpdbfile.site.name)
230 return True
232 # Called from GuiBulkImport to add a file or directory. Bulk import never monitors
233 def addBulkImportImportFileOrDir(self, inputPath, site="auto"):
234 """Add a file or directory for bulk import"""
235 # for windows platform, force os.walk variable to be unicode
236 # see fpdb-main post 9th July 2011
237 if self.config.posix:
238 pass
239 else:
240 inputPath = str(inputPath)
242 # TODO: only add sane files?
243 if os.path.isdir(inputPath):
244 for subdir in os.walk(inputPath):
245 for file in subdir[2]:
246 self.addImportFile(os.path.join(subdir[0], file), site=site)
247 return True
248 else:
249 return self.addImportFile(inputPath, site=site)
251 # Add a directory of files to filelist
252 # Only one import directory per site supported.
253 # dirlist is a hash of lists:
254 # dirlist{ 'PokerStars' => ["/path/to/import/", "filtername"] }
255 def addImportDirectory(self, dir, monitor=False, site=("default", "hh"), filter="passthrough"):
256 # gets called by GuiAutoImport.
257 # This should really be using os.walk
258 # http://docs.python.org/library/os.html
259 if os.path.isdir(dir):
260 if monitor is True:
261 self.monitor = True
262 self.dirlist[site] = [dir] + [filter]
264 # print "addImportDirectory: checking files in", dir
265 for subdir in os.walk(dir):
266 for file in subdir[2]:
267 filename = os.path.join(subdir[0], file)
268 # ignore symbolic links (Linux & Mac)
269 if os.path.islink(filename):
270 log.info(f"Ignoring symlink {filename}")
271 continue
272 if (time() - os.stat(filename).st_mtime) <= 43200: # look all files modded in the last 12 hours
273 # need long time because FTP in Win does not
274 # update the timestamp on the HH during session
275 self.addImportFile(filename, "auto")
276 else:
277 log.warning(("Attempted to add non-directory '%s' as an import directory") % str(dir))
279 def runImport(self):
280 """ "Run full import on self.filelist. This is called from GuiBulkImport.py"""
282 # Initial setup
283 start = datetime.datetime.now()
284 starttime = time()
285 log.info(
286 ("Started at %s -- %d files to import. indexes: %s")
287 % (start, len(self.filelist), self.settings["dropIndexes"])
288 )
289 if self.settings["dropIndexes"] == "auto":
290 self.settings["dropIndexes"] = self.calculate_auto2(self.database, 12.0, 500.0)
291 if "dropHudCache" in self.settings and self.settings["dropHudCache"] == "auto":
292 self.settings["dropHudCache"] = self.calculate_auto2(
293 self.database, 25.0, 500.0
294 ) # returns "drop"/"don't drop"
296 (totstored, totdups, totpartial, totskipped, toterrors) = self.importFiles(None)
298 # Tidying up after import
299 # if 'dropHudCache' in self.settings and self.settings['dropHudCache'] == 'drop':
300 # log.info(("rebuild_caches"))
301 # self.database.rebuild_caches()
302 # else:
303 # log.info(("runPostImport"))
304 self.runPostImport()
305 self.database.analyzeDB()
306 endtime = time()
307 return (totstored, totdups, totpartial, totskipped, toterrors, endtime - starttime)
309 # end def runImport
311 def runPostImport(self):
312 self.database.cleanUpTourneyTypes()
313 self.database.cleanUpWeeksMonths()
314 self.database.resetClean()
316 def importFiles(self, q):
317 """Read filenames in self.filelist and pass to despatcher."""
319 totstored = 0
320 totdups = 0
321 totpartial = 0
322 totskipped = 0
323 toterrors = 0
324 filecount = 0
325 fileerrorcount = 0
326 moveimportedfiles = False # TODO need to wire this into GUI and make it prettier
327 movefailedfiles = False # TODO and this too
329 # prepare progress popup window
330 ProgressDialog = ImportProgressDialog(len(self.filelist), self.parent)
331 ProgressDialog.resize(500, 200)
332 ProgressDialog.show()
334 for f in self.filelist:
335 filecount += 1
336 ProgressDialog.progress_update(f, str(self.database.getHandCount()))
338 (stored, duplicates, partial, skipped, errors, ttime) = self._import_despatch(self.filelist[f])
339 totstored += stored
340 totdups += duplicates
341 totpartial += partial
342 totskipped += skipped
343 toterrors += errors
345 if moveimportedfiles or movefailedfiles:
346 try:
347 if moveimportedfiles:
348 shutil.move(f, "c:\\fpdbimported\\%d-%s" % (filecount, os.path.basename(f[3:])))
349 except (shutil.Error, OSError) as e:
350 fileerrorcount += 1
351 log.error(f"Error moving imported file {f}: {e}")
352 if movefailedfiles:
353 try:
354 shutil.move(f, "c:\\fpdbfailed\\%d-%s" % (fileerrorcount, os.path.basename(f[3:])))
355 except (shutil.Error, OSError) as e:
356 log.error(f"Error moving failed file {f}: {e}")
358 self.logImport("bulk", f, stored, duplicates, partial, skipped, errors, ttime, self.filelist[f].fileId)
360 ProgressDialog.accept()
361 del ProgressDialog
363 return totstored, totdups, totpartial, totskipped, toterrors
365 # end def importFiles
367 def _import_despatch(self, fpdbfile):
368 stored, duplicates, partial, skipped, errors, ttime = 0, 0, 0, 0, 0, 0
369 if fpdbfile.ftype in ("hh", "both"):
370 (stored, duplicates, partial, skipped, errors, ttime) = self._import_hh_file(fpdbfile)
371 if fpdbfile.ftype == "summary":
372 (stored, duplicates, partial, skipped, errors, ttime) = self._import_summary_file(fpdbfile)
373 if fpdbfile.ftype == "both" and fpdbfile.path not in self.updatedsize:
374 self._import_summary_file(fpdbfile)
375 # pass
376 print("DEBUG: _import_summary_file.ttime: %.3f %s" % (ttime, fpdbfile.ftype))
377 return (stored, duplicates, partial, skipped, errors, ttime)
379 def calculate_auto2(self, db, scale, increment):
380 """A second heuristic to determine a reasonable value of drop/don't drop
381 This one adds up size of files to import to guess number of hands in them
382 Example values of scale and increment params might be 10 and 500 meaning
383 roughly: drop if importing more than 10% (100/scale) of hands in db or if
384 less than 500 hands in db"""
385 size_per_hand = 1300.0 # wag based on a PS 6-up FLHE file. Actual value not hugely important
386 # as values of scale and increment compensate for it anyway.
387 # decimal used to force float arithmetic
389 # get number of hands in db
390 if "handsInDB" not in self.settings:
391 try:
392 tmpcursor = db.get_cursor()
393 tmpcursor.execute("SELECT COUNT(1) FROM Hands;")
394 self.settings["handsInDB"] = tmpcursor.fetchone()[0]
395 except Exception as e:
396 log.error(f"Failed to retrieve hands count from database: {e}")
397 return "don't drop"
399 # add up size of import files
400 total_size = 0.0
401 for file in self.filelist:
402 if os.path.exists(file):
403 stat_info = os.stat(file)
404 total_size += stat_info.st_size
406 # if hands_in_db is zero or very low, we want to drop indexes, otherwise compare
407 # import size with db size somehow:
408 ret = "don't drop"
409 if self.settings["handsInDB"] < scale * (old_div(total_size, size_per_hand)) + increment:
410 ret = "drop"
411 # print "auto2: handsindb =", self.settings['handsInDB'], "total_size =", total_size, "size_per_hand =", \
412 # size_per_hand, "inc =", increment, "return:", ret
413 return ret
415 # Run import on updated files, then store latest update time. Called from GuiAutoImport.py
416 def runUpdated(self):
417 """Check for new files in monitored directories"""
418 for site, type in self.dirlist:
419 self.addImportDirectory(self.dirlist[(site, type)][0], False, (site, type), self.dirlist[(site, type)][1])
421 for f in self.filelist:
422 if os.path.exists(f):
423 stat_info = os.stat(f)
424 if f in self.updatedsize: # we should be able to assume that if we're in size, we're in time as well
425 if stat_info.st_size > self.updatedsize[f] or stat_info.st_mtime > self.updatedtime[f]:
426 try:
427 if not os.path.isdir(f):
428 self.caller.addText("\n" + os.path.basename(f))
429 log.debug("os.path.basename", os.path.basename(f))
430 log.debug("self.caller:", self.caller)
431 log.debug(os.path.basename(f))
432 except KeyError:
433 log.error("File '%s' seems to have disappeared" % f)
434 (stored, duplicates, partial, skipped, errors, ttime) = self._import_despatch(self.filelist[f])
435 self.logImport(
436 "auto", f, stored, duplicates, partial, skipped, errors, ttime, self.filelist[f].fileId
437 )
438 self.database.commit()
439 try:
440 if not os.path.isdir(f): # Note: This assumes that whatever calls us has an "addText" func
441 self.caller.addText(
442 " %d stored, %d duplicates, %d partial, %d skipped, %d errors (time = %f)"
443 % (stored, duplicates, partial, skipped, errors, ttime)
444 )
445 log.debug("self.caller2:", self.caller)
446 except KeyError: # TODO: Again, what error happens here? fix when we find out ..
447 pass
448 self.updatedsize[f] = stat_info.st_size
449 self.updatedtime[f] = time()
450 else:
451 if os.path.isdir(f) or (time() - stat_info.st_mtime) < 60:
452 self.updatedsize[f] = 0
453 self.updatedtime[f] = 0
454 else:
455 self.updatedsize[f] = stat_info.st_size
456 self.updatedtime[f] = time()
457 else:
458 self.removeFromFileList[f] = True
460 for file in self.removeFromFileList:
461 if file in self.filelist:
462 del self.filelist[file]
464 self.removeFromFileList = {}
465 self.database.rollback()
466 self.runPostImport()
468 def _import_hh_file(self, fpdbfile):
469 """Function for actual import of a hh file
470 This is now an internal function that should not be called directly."""
472 (stored, duplicates, partial, skipped, errors, ttime) = (0, 0, 0, 0, 0, time())
474 # Load filter, process file, pass returned filename to import_fpdb_file
475 log.info(f"Converting {fpdbfile.path}")
477 filter_name = fpdbfile.site.filter_name
478 mod = __import__(fpdbfile.site.hhc_fname)
479 obj = getattr(mod, filter_name, None)
480 if callable(obj):
481 if fpdbfile.path in self.pos_in_file:
482 idx = self.pos_in_file[fpdbfile.path]
483 else:
484 self.pos_in_file[fpdbfile.path], idx = 0, 0
486 hhc = obj(
487 self.config,
488 in_path=fpdbfile.path,
489 index=idx,
490 autostart=False,
491 starsArchive=fpdbfile.archive,
492 ftpArchive=fpdbfile.archive,
493 sitename=fpdbfile.site.name,
494 )
495 hhc.setAutoPop(self.mode == "auto")
496 hhc.start()
498 self.pos_in_file[fpdbfile.path] = hhc.getLastCharacterRead()
500 # Tally the results
501 partial = getattr(hhc, "numPartial")
502 skipped = getattr(hhc, "numSkipped")
503 errors = getattr(hhc, "numErrors")
504 stored = getattr(hhc, "numHands")
505 stored -= errors
506 stored -= partial
507 stored -= skipped
509 if stored > 0:
510 if self.caller:
511 self.progressNotify()
512 handlist = hhc.getProcessedHands()
513 self.database.resetBulkCache(True)
514 self.pos_in_file[fpdbfile.path] = hhc.getLastCharacterRead()
515 (phands, ahands, ihands, to_hud) = ([], [], [], [])
516 self.database.resetBulkCache()
518 ####Lock Placeholder####
519 for hand in handlist:
520 hand.prepInsert(self.database, printtest=self.settings["testData"])
521 ahands.append(hand)
522 self.database.commit()
523 ####Lock Placeholder####
525 for hand in ahands:
526 hand.assembleHand()
527 phands.append(hand)
529 ####Lock Placeholder####
530 backtrack = False
531 id = self.database.nextHandId()
532 for i in range(len(phands)):
533 doinsert = len(phands) == i + 1
534 hand = phands[i]
535 try:
536 id = hand.getHandId(self.database, id)
537 hand.updateSessionsCache(self.database, None, doinsert)
538 hand.insertHands(self.database, fpdbfile.fileId, doinsert, self.settings["testData"])
539 hand.updateCardsCache(self.database, None, doinsert)
540 hand.updatePositionsCache(self.database, None, doinsert)
541 hand.updateHudCache(self.database, doinsert)
542 hand.updateTourneyResults(self.database)
543 ihands.append(hand)
544 to_hud.append(hand.dbid_hands)
545 except FpdbHandDuplicate:
546 duplicates += 1
547 if doinsert and ihands:
548 backtrack = True
549 except Exception as e:
550 log.error(f"Importer._import_hh_file: '{fpdbfile.path}' Fatal error: '{e}'")
551 log.error(f"'{hand.handText[0:200]}'")
552 if doinsert and ihands:
553 backtrack = True
555 if backtrack:
556 hand = ihands[-1]
557 hp, hero = hand.handsplayers, hand.hero
558 hand.hero, self.database.hbulk, hand.handsplayers = (
559 0,
560 self.database.hbulk[:-1],
561 [],
562 ) # making sure we don't insert data from this hand
563 self.database.bbulk = [b for b in self.database.bbulk if hand.dbid_hands != b[0]]
564 hand.updateSessionsCache(self.database, None, doinsert)
565 hand.insertHands(self.database, fpdbfile.fileId, doinsert, self.settings["testData"])
566 hand.updateCardsCache(self.database, None, doinsert)
567 hand.updatePositionsCache(self.database, None, doinsert)
568 hand.updateHudCache(self.database, doinsert)
569 hand.handsplayers, hand.hero = hp, hero
571 self.database.commit()
572 ####Lock Placeholder####
574 for i in range(len(ihands)):
575 doinsert = len(ihands) == i + 1
576 hand = ihands[i]
577 hand.insertHandsPlayers(self.database, doinsert, self.settings["testData"])
578 hand.insertHandsActions(self.database, doinsert, self.settings["testData"])
579 hand.insertHandsStove(self.database, doinsert)
580 self.database.commit()
582 # pipe the Hands.id out to the HUD
583 if self.callHud:
584 if self.zmq_sender is None:
585 self.zmq_sender = ZMQSender()
586 for hid in list(to_hud):
587 try:
588 log.debug(f"Sending hand ID {hid} to HUD via socket")
589 self.zmq_sender.send_hand_id(hid)
590 except IOError as e:
591 log.error(f"Failed to send hand ID to HUD via socket: {e}")
593 # Cache HHC if enabled
594 if self.settings.get("cacheHHC", False):
595 self.handhistoryconverter = hhc
596 elif self.mode == "auto":
597 return (0, 0, partial, skipped, errors, time() - ttime)
599 stored -= duplicates
601 if stored > 0 and ihands[0].gametype["type"] == "tour":
602 if hhc.summaryInFile:
603 fpdbfile.ftype = "both"
605 ttime = time() - ttime
606 return (stored, duplicates, partial, skipped, errors, ttime)
608 def autoSummaryGrab(self, force=False):
609 for f, fpdbfile in list(self.filelist.items()):
610 stat_info = os.stat(f)
611 if ((time() - stat_info.st_mtime) > 300 or force) and fpdbfile.ftype == "both":
612 self._import_summary_file(fpdbfile)
613 fpdbfile.ftype = "hh"
615 def _import_summary_file(self, fpdbfile):
616 (stored, duplicates, partial, skipped, errors, ttime) = (0, 0, 0, 0, 0, time())
617 mod = __import__(fpdbfile.site.summary)
618 obj = getattr(mod, fpdbfile.site.summary, None)
619 if callable(obj):
620 if self.caller:
621 self.progressNotify()
622 summaryTexts = self.readFile(obj, fpdbfile.path, fpdbfile.site.name)
623 if summaryTexts is None:
624 log.error(
625 "Found: '%s' with 0 characters... skipping" % fpdbfile.path
626 ) # Fixed the typo (fpbdfile -> fpdbfile)
627 return (0, 0, 0, 0, 1, time()) # File had 0 characters
628 ####Lock Placeholder####
629 for j, summaryText in enumerate(summaryTexts, start=1):
630 try:
631 conv = obj(
632 db=self.database,
633 config=self.config,
634 siteName=fpdbfile.site.name,
635 summaryText=summaryText,
636 in_path=fpdbfile.path,
637 header=summaryTexts[0],
638 )
639 self.database.resetBulkCache(False)
640 conv.insertOrUpdate(printtest=self.settings["testData"])
641 except FpdbHandPartial:
642 partial += 1
643 except FpdbParseError:
644 log.error(f"Summary import parse error in file: {fpdbfile.path}")
645 errors += 1
646 if j != 1:
647 print(f"Finished importing {j}/{len(summaryTexts)} tournament summaries")
648 stored = j
649 ####Lock Placeholder####
650 ttime = time() - ttime
651 return (stored - errors - partial, duplicates, partial, skipped, errors, ttime)
653 def progressNotify(self):
654 "A callback to the interface while events are pending"
655 QCoreApplication.processEvents()
657 def readFile(self, obj, filename, site):
658 if filename.endswith(".xls") or filename.endswith(".xlsx") and xlrd:
659 obj.hhtype = "xls"
660 if site == "PokerStars":
661 tourNoField = "Tourney"
662 else:
663 tourNoField = "tournament key"
664 summaryTexts = obj.summaries_from_excel(filename, tourNoField)
665 else:
666 foabs = obj.readFile(obj, filename)
667 if foabs is None:
668 return None
669 re_Split = obj.getSplitRe(obj, foabs)
670 summaryTexts = re.split(re_Split, foabs)
671 # Summary identified but not split
672 if len(summaryTexts) == 1:
673 return summaryTexts
674 else:
675 # The summary files tend to have a header
676 # Remove the first entry if it has < 150 characters
677 if len(summaryTexts) > 1 and len(summaryTexts[0]) <= 150:
678 del summaryTexts[0]
679 log.warn(("TourneyImport: Removing text < 150 characters from start of file"))
681 # Sometimes the summary files also have a footer
682 # Remove the last entry if it has < 100 characters
683 if len(summaryTexts) > 1 and len(summaryTexts[-1]) <= 100:
684 summaryTexts.pop()
685 log.warn(("TourneyImport: Removing text < 100 characters from end of file"))
686 return summaryTexts
688 def __del__(self):
689 if hasattr(self, "zmq_sender"):
690 self.zmq_sender.close()
693class ImportProgressDialog(QDialog):
694 """
695 Popup window to show progress
697 Init method sets up total number of expected iterations
698 If no parent is passed to init, command line
699 mode assumed, and does not create a progress bar
700 """
702 def progress_update(self, filename, handcount):
703 self.fraction += 1
704 # update total if fraction exceeds expected total number of iterations
705 if self.fraction > self.total:
706 self.total = self.fraction
707 self.pbar.setRange(0, self.total)
709 self.pbar.setValue(self.fraction)
711 self.handcount.setText(("Database Statistics") + " - " + ("Number of Hands:") + " " + handcount)
713 now = datetime.datetime.now()
714 now_formatted = now.strftime("%H:%M:%S")
715 self.progresstext.setText(now_formatted + " - " + ("Importing") + " " + filename + "\n")
717 def __init__(self, total, parent):
718 if parent is None:
719 return
720 QDialog.__init__(self, parent)
722 self.fraction = 0
723 self.total = total
724 self.setWindowTitle(("Importing"))
726 self.setLayout(QVBoxLayout())
728 self.pbar = QProgressBar()
729 self.pbar.setRange(0, total)
730 self.layout().addWidget(self.pbar)
732 self.handcount = QLabel()
733 self.handcount.setWordWrap(True)
734 self.layout().addWidget(self.handcount)
736 self.progresstext = QLabel()
737 self.progresstext.setWordWrap(True)
738 self.layout().addWidget(self.progresstext)