Skip to content

Commit 93bfd2a

Browse files
Preprocessing of template resources
src or href urls are augmented with "?UNIQUE_ID" of the respective resource. Testing is done only for 404 HTML responses (because at this point we have other content checking kiwix-serve tests).
1 parent 4cce4dc commit 93bfd2a

File tree

4 files changed

+128
-12
lines changed

4 files changed

+128
-12
lines changed

scripts/kiwix-resources

+106
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,106 @@
1+
#!/usr/bin/env python3
2+
3+
'''
4+
Copyright 2022 Veloman Yunkan <[email protected]>
5+
6+
This program is free software; you can redistribute it and/or modify
7+
it under the terms of the GNU General Public License as published by
8+
the Free Software Foundation; either version 3 of the License, or any
9+
later version.
10+
11+
This program is distributed in the hope that it will be useful, but
12+
WITHOUT ANY WARRANTY; without even the implied warranty of
13+
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14+
General Public License for more details.
15+
16+
You should have received a copy of the GNU General Public License
17+
along with this program; if not, write to the Free Software
18+
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
19+
02110-1301, USA.
20+
'''
21+
22+
import argparse
23+
import hashlib
24+
import os.path
25+
import re
26+
27+
def read_resource_file(resource_file_path):
28+
with open(resource_file_path, 'r') as f:
29+
return [line.strip() for line in f]
30+
31+
def list_resources(resource_file_path):
32+
for resource_path in read_resource_file(resource_file_path):
33+
print(resource_path)
34+
35+
def get_resource_revision(base_dir, resource_path):
36+
with open(os.path.join(base_dir, resource_path), 'rb') as f:
37+
return hashlib.sha1(f.read()).hexdigest()[:8]
38+
39+
resource_revisions = {}
40+
41+
def fill_resource_revisions(resource_file_path):
42+
base_dir = os.path.dirname(os.path.realpath(resource_file_path))
43+
for resource in read_resource_file(resource_file_path):
44+
resource_revisions[resource] = get_resource_revision(base_dir, resource)
45+
46+
src_or_href_pattern=r'(src|href)="\{\{root\}\}/(skin/[^"]+)"'
47+
def add_resource_revision(resource_matchobj):
48+
attr = resource_matchobj.group(1)
49+
resource = resource_matchobj.group(2)
50+
if resource in resource_revisions:
51+
uniqueid = '?' + resource_revisions[resource]
52+
else:
53+
uniqueid = ''
54+
return attr + r'="{{root}}/' + resource + uniqueid + '"'
55+
56+
def preprocess_template(srcpath, dstpath):
57+
with open(srcpath, 'r') as source:
58+
with open(dstpath, 'w') as target:
59+
for line in source:
60+
line = re.sub(src_or_href_pattern, add_resource_revision, line)
61+
print(line, end='', file=target)
62+
63+
def symlink_resource(src, resource_path):
64+
if os.path.exists(resource_path):
65+
if os.path.islink(resource_path) and os.readlink(resource_path) == src:
66+
return
67+
os.remove(resource_path)
68+
os.symlink(src, resource_path)
69+
70+
def preprocess_resource(srcdir, resource_path, outdir):
71+
resource_dir = os.path.dirname(resource_path)
72+
if resource_dir != '':
73+
os.makedirs(os.path.join(outdir, resource_dir), exist_ok=True)
74+
srcpath = os.path.join(srcdir, resource_path)
75+
outpath = os.path.join(outdir, resource_path)
76+
if resource_path.startswith('templates/'):
77+
preprocess_template(srcpath, outpath)
78+
else:
79+
symlink_resource(srcpath, outpath)
80+
81+
def copy_file(src_path, dst_path):
82+
with open(src_path, 'rb') as src:
83+
with open(dst_path, 'wb') as dst:
84+
dst.write(src.read())
85+
86+
def preprocess_resources(resource_file_path, outdir):
87+
base_dir = os.path.dirname(os.path.realpath(resource_file_path))
88+
resource_filename = os.path.basename(resource_file_path)
89+
for resource in read_resource_file(resource_file_path):
90+
preprocess_resource(base_dir, resource, outdir)
91+
copy_file(resource_file_path, os.path.join(outdir, resource_filename))
92+
93+
if __name__ == "__main__":
94+
parser = argparse.ArgumentParser()
95+
commands = parser.add_mutually_exclusive_group()
96+
commands.add_argument('--list-all', action='store_true')
97+
commands.add_argument('--preprocess', action='store_true')
98+
parser.add_argument('--outdir')
99+
parser.add_argument('resource_file')
100+
args = parser.parse_args()
101+
102+
if args.list_all:
103+
list_resources(args.resource_file)
104+
elif args.preprocess:
105+
fill_resource_revisions(args.resource_file)
106+
preprocess_resources(args.resource_file, args.outdir)

scripts/meson.build

+1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11

2+
res_manager = find_program('kiwix-resources')
23
res_compiler = find_program('kiwix-compile-resources')
34

45
install_data(res_compiler.path(), install_dir:get_option('bindir'))

static/meson.build

+14-5
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,25 @@
1-
resource_files = run_command(find_program('python3'),
2-
'-c',
3-
'import sys; f=open(sys.argv[1]); print(f.read())',
1+
resource_files = run_command(res_manager,
2+
'--list-all',
43
files('resources_list.txt')
54
).stdout().strip().split('\n')
65

7-
lib_resources = custom_target('resources',
6+
preprocessed_resources = custom_target('preprocessed_resource_files',
87
input: 'resources_list.txt',
8+
output: ['resources_list.txt'],
9+
command:[res_manager,
10+
'--preprocess',
11+
'--outdir', '@OUTDIR@',
12+
'@INPUT@'],
13+
depend_files: resource_files
14+
)
15+
16+
lib_resources = custom_target('resources',
17+
input: preprocessed_resources,
918
output: ['kiwixlib-resources.cpp', 'kiwixlib-resources.h'],
1019
command:[res_compiler,
1120
'--cxxfile', '@OUTPUT0@',
1221
'--hfile', '@OUTPUT1@',
1322
'--source_dir', '@OUTDIR@',
1423
'@INPUT@'],
15-
depend_files: resource_files
24+
depends: preprocessed_resources
1625
)

test/server.cpp

+7-7
Original file line numberDiff line numberDiff line change
@@ -359,12 +359,12 @@ std::string TestContentIn404HtmlResponse::expectedResponse() const
359359
<head>
360360
<meta content="text/html;charset=UTF-8" http-equiv="content-type" />
361361
<title>Content not found</title>
362-
<link type="root" href="/ROOT"><link type="text/css" href="/ROOT/skin/jquery-ui/jquery-ui.min.css" rel="Stylesheet" />
363-
<link type="text/css" href="/ROOT/skin/jquery-ui/jquery-ui.theme.min.css" rel="Stylesheet" />
364-
<link type="text/css" href="/ROOT/skin/taskbar.css" rel="Stylesheet" />
365-
<script type="text/javascript" src="/ROOT/skin/jquery-ui/external/jquery/jquery.js" defer></script>
366-
<script type="text/javascript" src="/ROOT/skin/jquery-ui/jquery-ui.min.js" defer></script>
367-
<script type="text/javascript" src="/ROOT/skin/taskbar.js" defer></script>
362+
<link type="root" href="/ROOT"><link type="text/css" href="/ROOT/skin/jquery-ui/jquery-ui.min.css?e1de77b3" rel="Stylesheet" />
363+
<link type="text/css" href="/ROOT/skin/jquery-ui/jquery-ui.theme.min.css?2a5841f9" rel="Stylesheet" />
364+
<link type="text/css" href="/ROOT/skin/taskbar.css?49365e9c" rel="Stylesheet" />
365+
<script type="text/javascript" src="/ROOT/skin/jquery-ui/external/jquery/jquery.js?1d85f0f3" defer></script>
366+
<script type="text/javascript" src="/ROOT/skin/jquery-ui/jquery-ui.min.js?d927c2ff" defer></script>
367+
<script type="text/javascript" src="/ROOT/skin/taskbar.js?3e1a2497" defer></script>
368368
</head>
369369
<body><span class="kiwix">
370370
<span id="kiwixtoolbar" class="ui-widget-header">
@@ -380,7 +380,7 @@ std::string TestContentIn404HtmlResponse::expectedResponse() const
380380
R"FRAG( </form>
381381
</div>
382382
<input type="checkbox" id="kiwix_button_show_toggle">
383-
<label for="kiwix_button_show_toggle"><img src="/ROOT/skin/caret.png" alt=""></label>
383+
<label for="kiwix_button_show_toggle"><img src="/ROOT/skin/caret.png?22b942b4" alt=""></label>
384384
<div class="kiwix_button_cont">
385385
<a id="kiwix_serve_taskbar_library_button" title="Go to welcome page" aria-label="Go to welcome page" href="/ROOT/"><button>&#x1f3e0;</button></a>
386386
)FRAG",

0 commit comments

Comments
 (0)