From 0d2d6c216eee8cb8d83fbb876e97f7bdb6387292 Mon Sep 17 00:00:00 2001 From: coder Date: Sun, 26 Apr 2020 00:11:36 +1000 Subject: [PATCH] Formatted python code using flake8 w/ default configuration --- .../DataStructureManupulations.py | 42 ++-- generator/GeneralFunctions/InputOutput.py | 8 +- generator/GeneralFunctions/Network.py | 2 +- generator/GeneralFunctions/OS.py | 2 + generator/main.py | 226 +++++++++--------- 5 files changed, 155 insertions(+), 125 deletions(-) diff --git a/generator/GeneralFunctions/DataStructureManupulations.py b/generator/GeneralFunctions/DataStructureManupulations.py index 5187a05..da30347 100644 --- a/generator/GeneralFunctions/DataStructureManupulations.py +++ b/generator/GeneralFunctions/DataStructureManupulations.py @@ -8,6 +8,7 @@ else: PathSlash = '/' FileProtocol = "file:" + 2*PathSlash + def ConvertToStandardPathFormat(Path): """ Example, Input: '"file:///some/path/somefile.extension" @@ -18,6 +19,7 @@ def ConvertToStandardPathFormat(Path): Path = Path[len(FileProtocol):] return Path + def GetTextAfter(Text, ReadlinesTextFile): for Lines in range(len(ReadlinesTextFile)): Line = ReadlinesTextFile[Lines].strip('\n') @@ -25,18 +27,21 @@ def GetTextAfter(Text, ReadlinesTextFile): return Line[len(Text):] return '' + def SingleQuoteString(String): if len(String) > 0: if String[0] != '\'' or String[-1] != '\'': String = '\'' + String + '\'' return String + def DoubleQuoteString(String): if len(String) > 0: if String[0] != '"' or String[-1] != '"': String = '"' + String + '"' return String + def ListIntoString(List, QuoteItems=0, Seprator=' '): if QuoteItems == 2: for i in range(len(List)): @@ -49,36 +54,42 @@ def ListIntoString(List, QuoteItems=0, Seprator=' '): Stringoflist = (Seprator).join(List) return Stringoflist + # strip=0 => remove both ' & ", 1 => remove ', 2 => remove " def UnquoteString(String, strip=0): while True: if ( - strip != 2 - and String.startswith('"') - and String.endswith('"')): + strip != 2 and + String.startswith('"') and + String.endswith('"') + ): String = String.strip('"') elif ( strip != 1 and String.startswith("'") - and String.endswith("'")): + and String.endswith("'") + ): String = String.strip("'") else: break return String + def StandardVariableName(Variable): Variable = Variable.casefold() Variable = Variable.replace('_', '').replace(' ', '') return Variable -#def DictionaryToJsonStr(Dict, BaseIndentation=0): - #BI = '\t'*BaseIndentation - #JsonStr = BI+'{\n' - #for k, v in Dict.items(): - #JsonStr += BI+'\t"'+k+'" : "'+v+'",\n' - #JsonStr = JsonStr[:-2] - #JsonStr += '\n'+BI+'}' - #return JsonStr +""" +def DictionaryToJsonStr(Dict, BaseIndentation=0): + BI = '\t'*BaseIndentation + JsonStr = BI+'{\n' + for k, v in Dict.items(): + JsonStr += BI+'\t"'+k+'" : "'+v+'",\n' + JsonStr = JsonStr[:-2] + JsonStr += '\n'+BI+'}' + return JsonStr +""" def StringToKeyValuePair(String, Seprator): SepratorAt = String.find(Seprator) @@ -89,14 +100,17 @@ def StringToKeyValuePair(String, Seprator): else: return "", String + def FormatStrForDictinary(String): String = String.strip(" \n\r") return UnquoteString(String) + def StrListToDictionary( List, - Seprator = '=', - FormatFunction = FormatStrForDictinary): + Seprator='=', + FormatFunction=FormatStrForDictinary + ): Dictionary = {} for i in List: k, v = StringToKeyValuePair(i, Seprator) diff --git a/generator/GeneralFunctions/InputOutput.py b/generator/GeneralFunctions/InputOutput.py index a503596..b2138fa 100644 --- a/generator/GeneralFunctions/InputOutput.py +++ b/generator/GeneralFunctions/InputOutput.py @@ -2,6 +2,7 @@ from os.path import isfile as DoesFileExist from .DataStructureManupulations import ConvertToStandardPathFormat + def ReadTextFile(FilePath): FilePath = ConvertToStandardPathFormat(FilePath) if DoesFileExist(FilePath) is True: @@ -11,11 +12,13 @@ def ReadTextFile(FilePath): return ReadFile else: return '' - + + def ReadlinesTextFile(FilePath): String = ReadTextFile(FilePath) return String.split('\n') + def WriteTextFiles(FilePath, Text): if type(Text) != str: Text = '\n'.join(Text) @@ -23,7 +26,8 @@ def WriteTextFiles(FilePath, Text): File.write(Text) File.close() + def WriteBinaryToFile(Filepath, Data): - File=open(Filepath, 'wb') + File = open(Filepath, 'wb') File.write(Data) File.close() diff --git a/generator/GeneralFunctions/Network.py b/generator/GeneralFunctions/Network.py index c506405..2f469f2 100644 --- a/generator/GeneralFunctions/Network.py +++ b/generator/GeneralFunctions/Network.py @@ -3,6 +3,6 @@ from urllib.request import urlopen HttpsContext = ssl.create_default_context() + def Download(Url): return urlopen(Url, context=HttpsContext).read() - diff --git a/generator/GeneralFunctions/OS.py b/generator/GeneralFunctions/OS.py index eb250db..152527e 100644 --- a/generator/GeneralFunctions/OS.py +++ b/generator/GeneralFunctions/OS.py @@ -1,11 +1,13 @@ import os + def CallFuncInDir(Directory, Function, *args, **kwArgs): CurrentDir = os.getcwd() os.chdir(Directory) Function(*args, **kwArgs) os.chdir(CurrentDir) + # return True if operation succesful and False if failed def CreateDir(Directory): if not os.path.isfile(Directory): diff --git a/generator/main.py b/generator/main.py index 0293c23..0c1e3b7 100644 --- a/generator/main.py +++ b/generator/main.py @@ -6,10 +6,6 @@ All sub-directories of bundles directory will be scanned for activity bundles i.e. .xo files. """ -""" FIXME: paths hard coded unix style & most likely will not work on winodws. -Use code written for IMM to handle it. -""" - import glob import json import os @@ -19,8 +15,6 @@ from urllib.parse import quote as strToHtmlFmt import zipfile from GeneralFunctions.DataStructureManupulations import ( - GetTextAfter, - UnquoteString, StrListToDictionary ) from GeneralFunctions.InputOutput import ( @@ -32,15 +26,21 @@ from GeneralFunctions.OS import ( CreateDir ) + +""" FIXME: paths hard coded unix style & most likely will not work on winodws. +Use code written for IMM to handle it. +""" + + class extractData: - + def findInfoFiles(self, bundle): infoFiles = [] for File in bundle.namelist(): if File.endswith("activity/activity.info"): infoFiles.append(File) return infoFiles - + def copyBundle(self, bundleSrc, activityName): CallFuncInDir( self.websiteDir, @@ -48,31 +48,31 @@ class extractData: self.bundlesDir+bundleSrc, "bundles/"+activityName+".xo" ) - + def createDirectories(self): assert(CreateDir("app")) assert(CreateDir("icons")) assert(CreateDir("bundles")) assert(CreateDir("js")) - + def extractActivityInfo(self, infoFilePath, zipFile): - infoList, infoDict = [], {} + infoList = [] infoList = zipFile.read( infoFilePath).decode("utf-8").split('\n') return StrListToDictionary(infoList) - + def extractInfoAndIconFromBundles(self): for bundlePath in self.activityBundles: bundle = zipfile.ZipFile(bundlePath, "r") - + infoFiles = self.findInfoFiles(bundle) if len(infoFiles) != 1: self.bundlesNotExactlyOneInfoFile.append(bundlePath) else: - + infoDict = self.extractActivityInfo(infoFiles[0], bundle) self.bundlesInfoList.append(infoDict) - + # FIXME: create seprate function for it # extract and copy icon activityName = infoDict.get("name") @@ -85,9 +85,9 @@ class extractData: if iconAbsolutePath in bundle.namelist(): icon = bundle.read(iconAbsolutePath) iconPath = ( - "icons/"+ + "icons/" + activityName - +".svg" + + ".svg" ) CallFuncInDir( self.websiteDir, @@ -98,115 +98,122 @@ class extractData: else: # Conitnue without icon since non-fatal error self.iconErroredBundles.append(bundlePath) - + bundle.close() # FIXME: uncomment below function. # Disabled sometime during devlopment as time consuming self.copyBundle(bundlePath, activityName) bundle.close() - + def generateAppsHtmlPages(self): iconsDir = "../icons/" bundlesDir = "../bundles/" for appInfo in self.indexDictList: pathName = strToHtmlFmt(appInfo["name"], safe='') - + html = ( - '\n\n\n'+appInfo["name"]+ + '<!DOCTYPE html>\n<html>\n<head>\n<title>' + appInfo["name"] + '\n\n\n\n\n' - '\n

'+appInfo["name"]+'

\n

\n
' - '

Summary

\n

'+appInfo["summary"]+ - '

\n
\n

Description

\n

'+ - appInfo["description"]+'

\n
\n

Tags' + '\n

' + appInfo["name"] + '

\n

\n' + '

Summary

\n

' + appInfo["summary"] + + '

\n
\n

Description

\n

' + + appInfo["description"] + '

\n
\n

Tags' '

\n
    \n' ) for tag in appInfo["tags"]: - html += '
  • '+tag+'
    • \n' + html += '
  • ' + tag + '
    • \n' html += ( - '
\n
\n

Download

\n\n' + '\n

\n

Download

\n\n' ) - - WriteTextFiles("./app/"+appInfo["name"]+".html", html) + + WriteTextFiles("./app/" + appInfo["name"] + ".html", html) """ Only those which are specified in map will be added to index. - If an entry or value does not exist in infoJSON than emprty entry will + If an entry or value does not exist in infoJSON than emprty entry will be created for it. appends keys rather than replacing where mutiple map to same """ - def generateIndex(self, - infoToIndexMap = { - "name" : ("name", str), - "summary" : ("summary", str), - "description" : ("description", str), - "tag" : ("tags", list), - "tags" : ("tags", list), - "categories" : ("tags", list), - "category" : ("tags", list) - }): - unexpectedInputError = ( - "main.py generateIndex() : expect only str, list or tuple as " - "kwargs -> value[1] but found " - ) - - i2IMap = infoToIndexMap - self.indexDictList = [] - - for obj in json.loads(self.infoJson): - indexDict = {} - for k, v in obj.items(): - if k in i2IMap: - - # add new entry/key to app index - if k not in indexDict: - if i2IMap[k][1] == str: - indexDict[i2IMap[k][0]] = v - elif (i2IMap[k][1] == list - or i2IMap[k][1] == tuple): - if v.find(';') >= 0: - indexDict[i2IMap[k][0]] = v.split(';') - else: - indexDict[i2IMap[k][0]] = v.split() - - # Append to existing entry/key to app index - else: - if i2IMap[k][1] == str: - indexDict[i2IMap[k][0]] += ' '+v - elif (i2IMap[k][1] == list - or i2IMap[k][1] == tuple): - if v.find(';') >= 0: - indexDict[i2IMap[k][0]] += v.split(';') - else: - indexDict[i2IMap[k][0]] += v.split() + def generateIndex( + self, + infoToIndexMap={ + "name": ("name", str), + "summary": ("summary", str), + "description": ("description", str), + "tag": ("tags", list), + "tags": ("tags", list), + "categories": ("tags", list), + "category": ("tags", list) + } + ): + unexpectedInputError = ( + "main.py generateIndex() : expect only str, list or tuple as " + "kwargs -> value[1] but found " + ) + + i2IMap = infoToIndexMap + self.indexDictList = [] + + for obj in json.loads(self.infoJson): + indexDict = {} + for k, v in obj.items(): + if k in i2IMap: + + # add new entry/key to app index + if k not in indexDict: + if i2IMap[k][1] == str: + indexDict[i2IMap[k][0]] = v + elif (i2IMap[k][1] == list + or i2IMap[k][1] == tuple): + if v.find(';') >= 0: + indexDict[i2IMap[k][0]] = v.split(';') + else: + indexDict[i2IMap[k][0]] = v.split() + + # Append to existing entry/key to app index else: - print(unexpectedInputError, i2IMap[k][1]) + if i2IMap[k][1] == str: + indexDict[i2IMap[k][0]] += ' '+v + elif (i2IMap[k][1] == list + or i2IMap[k][1] == tuple): + if v.find(';') >= 0: + indexDict[i2IMap[k][0]] += v.split(';') + else: + indexDict[i2IMap[k][0]] += v.split() + else: + print(unexpectedInputError, i2IMap[k][1]) + sys.exit(1) + + # Create entry/key with empty value for keys not present + # in activty.info + for k, v in i2IMap.items(): + if v[0] not in indexDict: + if v[1] == str: + indexDict[v[0]] = "" + elif (v[1] == list or v[1] == tuple): + indexDict[v[0]] = () + else: + print(unexpectedInputError, v[1]) sys.exit(1) - - # Create entry/key with empty value for keys not present - # in activty.info - for k ,v in i2IMap.items(): - if v[0] not in indexDict: - if v[1] == str: - indexDict[v[0]] = "" - elif (v[1] == list or v[1] == tuple): - indexDict[v[0]] = () - else: - print(unexpectedInputError, v[1]) - sys.exit(1) - - self.indexDictList.append(indexDict) - self.indexJs = "search.assignIndex("+json.dumps(self.indexDictList, indent=4)+")" - + + self.indexDictList.append(indexDict) + self.indexJs = ( + "search.assignIndex(" + + json.dumps(self.indexDictList, indent=4) + + ")" + ) + def generateInfoJson(self): self.infoJson = json.dumps(self.bundlesInfoList, indent=4) - + def __init__(self, bundlesDir, websiteDir): """ FIXME: WARNING:: some files may be missing such as some app may not have icon (use a placeholder icon for them) Some bundles are not succesfully processed (html page + bundle copy) - but are not in error logs as well. There are 495 bundles in + but are not in error logs as well. There are 495 bundles in Tony's repo, 479 sucessfully processed but showing fatal error of 12 only, i.e. 4 missing. """ @@ -220,27 +227,27 @@ class extractData: self.indexDictList = [] self.erroredBundles = [] self.iconErroredBundles = [] - + CallFuncInDir(self.websiteDir, self.createDirectories) - + os.chdir(bundlesDir) self.activityBundles = glob.glob("**/*.xo", recursive=True) - + self.purgeBundlesNotZipFile() - + self.extractInfoAndIconFromBundles() - + self.generateInfoJson() - + self.generateIndex() - + os.chdir(websiteDir) - + self.generateAppsHtmlPages() self.writeFiles() - - #self.copyBundles() + + #self.copyBundles() def purgeBundlesNotZipFile(self): activityBundles = [] @@ -250,13 +257,13 @@ class extractData: else: self.bundlesNotZipFiles.append(bundle) self.activityBundles = activityBundles - + def writeFiles(self): """ Files which are not continously written during the process Eg. Html, icon and bundles are written while processing each bundle """ WriteTextFiles("info.json", self.infoJson) - WriteTextFiles("./js/index.js",self.indexJs) + WriteTextFiles("./js/index.js", self.indexJs) WriteTextFiles( "bundlesNotExactlyOneInfoFile.txt", self.bundlesNotExactlyOneInfoFile @@ -265,6 +272,7 @@ class extractData: WriteTextFiles("erroredBundles.txt", self.erroredBundles) WriteTextFiles("iconErroredBundles.txt", self.iconErroredBundles) + def processArguments(): variables = {} if len(sys.argv) == 3: @@ -282,9 +290,11 @@ def processArguments(): sys.exit(1) return variables + def main(): variables = processArguments() extractData(variables["bundlesDir"], variables["websiteDir"]) + if __name__ == "__main__": - main(); + main()