1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
|
#!/usr/bin/python
import sys, getopt, os, shutil, pprint
import xml.etree.ElementTree as ET
from zipfile import ZipFile
def main(argv):
inputdir = ''
outputdir = ''
extracted_files_dir_by_user = ''
#read the arguments
try:
opts, args = getopt.getopt(argv,"hi:o:e:",["idir=","odir="])
except getopt.GetoptError:
print ('analyze.py -i <inputdir> -o <outputdir>')
sys.exit(2)
for opt, arg in opts:
if opt == '-h':
print ('analyze.py -i <inputdir> -o <outputdir>')
sys.exit()
elif opt == '-e':
extracted_files_dir_by_user = arg
elif opt in ("-i", "--idir"):
inputdir = arg
elif opt in ("-o", "--odir"):
outputdir = arg
# holds the result structer of analyze
result_list = []
if(extracted_files_dir_by_user == ''):
# use default directory path for extracted ooxml files.
extracted_files_dir = os.path.join(outputdir, 'extractedfiles')
extract_files(inputdir, extracted_files_dir)
# create seperate result files for each ooxml document as <document name>.result in output directory
for ext_dir in get_list_of_subdir(extracted_files_dir):
i = ext_dir.rfind('/')
sub_result_name = ext_dir[i+1:] + ".result"
sub_result_list = []
count_elements(ext_dir, sub_result_list)
sub_result_path = os.path.join(outputdir, sub_result_name)
# sort the result sub list according to tag names
sub_result_list = sorted(sub_result_list, key=lambda x: list(x[0].keys())[0], reverse=False)
with open(sub_result_path, "w") as log_file:
pprint.pprint(sub_result_list, log_file)
else:
# use user defined directory path for extracted ooxml files.
count_elements(extracted_files_dir_by_user, result_list)
# unzip all ooxml files into the given path
def extract_files(inputdir, extracted_files_dir):
# clean extracted files directory firstly
if(os.path.exists(extracted_files_dir)):
shutil.rmtree(extracted_files_dir)
# unzip files into the extracted files directory
for filename in os.listdir(inputdir):
if (filename.endswith(".pptx") or \
filename.endswith(".docx") or \
filename.endswith(".xlsx")) and not \
filename.startswith("~"):
filepath = os.path.join(inputdir, filename)
extracted_file_path = os.path.join(extracted_files_dir, filename)
with ZipFile(filepath) as zipObj:
zipObj.extractall(extracted_file_path)
else:
continue
# get key of value in dictionary
def get_key(val, dict):
for key, value in dict.items():
if val == value:
return str(key)
return ''
# replace curlybrace namespaces with the shorten ones
def replace_namespace_with_alias(filename, element):
namespaces = dict([node for _, node in ET.iterparse(filename, events=['start-ns'])])
i = element.find('}')
if i>=0:
element_ns = element[1:i]
element_ns_alias = get_key(element_ns, namespaces)
if element_ns_alias !='':
element = element.replace("{" + element_ns + "}", element_ns_alias + ":")
else:
element = element.replace("{" + element_ns + "}", "")
return element
def is_file_in_accepted_files(filename):
if(filename.endswith("[Content_Types].xml") or \
filename.endswith("docProps/custom.xml") or \
filename.endswith("docProps/app.xml") or \
filename.endswith("presentation.xml") or \
filename.endswith("viewProps.xml") or \
filename.endswith("tableStyles.xml") or \
filename.endswith("presProps.xml") or \
"ppt/slideLayouts" in filename or \
"ppt/slideMasters" in filename or \
"ppt/theme" in filename or \
filename.endswith("docProps/core.xml") or not \
filename.endswith(".xml")):
return False
return True
# counts tags, attribute names and values of xmls
def count_elements(extracted_files_dir, result_list):
# make sure if extracted files directory exist
if not (os.path.exists(extracted_files_dir)):
print("Extracted files directory is not exist")
return
list_of_files = get_list_of_files(extracted_files_dir)
# parse xmls and count elements
for xmlfile in list_of_files:
if not is_file_in_accepted_files(xmlfile):
continue
print(xmlfile)
tree = ET.parse(xmlfile)
root = tree.getroot()
# start to count
for child in root.iter():
tag = replace_namespace_with_alias(xmlfile, child.tag)
tag_idx = get_index_of_tag(tag, result_list)
# count tags
if (tag_idx == -1):
tmp_list = [{tag: 1},{},{},{}]
result_list.append(tmp_list)
else:
result_list[tag_idx][0][tag] += 1
# count attribute names and values of current tag
for attr_name, attr_value in child.attrib.items():
attr_name = replace_namespace_with_alias(xmlfile, attr_name)
if not attr_name in result_list[tag_idx][1].keys():
result_list[tag_idx][1][attr_name] = 1
else:
result_list[tag_idx][1][attr_name] +=1
if not attr_value in result_list[tag_idx][2].keys():
result_list[tag_idx][2][attr_value] = 1
else:
result_list[tag_idx][2][attr_value] +=1
if not (str(child.text) == "None"):
if not child.text in result_list[tag_idx][3].keys():
result_list[tag_idx][3][child.text] = 1
else:
result_list[tag_idx][3][child.text] += 1
# gets the position of "tag" element in result list. If element is not exist,
# return -1 that points the last index of the list.
def get_index_of_tag(tag, result_list):
for idx, tag_list in enumerate(result_list):
if tag in tag_list[0].keys():
return idx
return -1
# list all xmls in extracted files directory
def get_list_of_files(directory_name):
list_of_file = os.listdir(directory_name)
all_files = list()
for filename in list_of_file:
full_path = os.path.join(directory_name, filename)
if os.path.isdir(full_path):
all_files = all_files + get_list_of_files(full_path)
else:
all_files.append(full_path)
return all_files
def get_list_of_subdir(directory_name):
list_of_file = os.listdir(directory_name)
subdirs = list()
for filename in list_of_file:
full_path = os.path.join(directory_name, filename)
if os.path.isdir(full_path):
subdirs.append(full_path)
return subdirs
if __name__ == "__main__":
main(sys.argv[1:])
|
