Skip to content

Commit b4f7dfa

Browse files
mgautierfrmgautierfr
authored
Merge pull request #553 from kiwix/catalog_languages_endpoint
2 parents 49322f5 + ab30957 commit b4f7dfa

12 files changed

+233
-60
lines changed

‎include/library.h

+13
Original file line numberDiff line numberDiff line change
@@ -154,6 +154,7 @@ class Library
154154

155155
public:
156156
typedef std::vector<std::string> BookIdCollection;
157+
typedef std::map<std::string, int> AttributeCounts;
157158

158159
public:
159160
Library();
@@ -242,6 +243,13 @@ class Library
242243
*/
243244
std::vector<std::string> getBooksLanguages() const;
244245

246+
/**
247+
* Get all languagues of the books in the library with counts.
248+
*
249+
* @return A list of languages with the count of books in each language.
250+
*/
251+
AttributeCounts getBooksLanguagesWithCounts() const;
252+
245253
/**
246254
* Get all categories of the books in the library.
247255
*
@@ -341,7 +349,12 @@ class Library
341349
friend class OPDSDumper;
342350
friend class libXMLDumper;
343351

352+
private: // types
353+
typedef const std::string& (Book::*BookStrPropMemFn)() const;
354+
344355
private: // functions
356+
AttributeCounts getBookAttributeCounts(BookStrPropMemFn p) const;
357+
std::vector<std::string> getBookPropValueSet(BookStrPropMemFn p) const;
345358
BookIdCollection filterViaBookDB(const Filter& filter) const;
346359
void updateBookDB(const Book& book);
347360
};

‎include/opds_dumper.h

+8-2
Original file line numberDiff line numberDiff line change
@@ -66,10 +66,16 @@ class OPDSDumper
6666
/**
6767
* Dump the categories OPDS feed.
6868
*
69-
* @param categories list of category names
7069
* @return The OPDS feed.
7170
*/
72-
std::string categoriesOPDSFeed(const std::vector<std::string>& categories) const;
71+
std::string categoriesOPDSFeed() const;
72+
73+
/**
74+
* Dump the languages OPDS feed.
75+
*
76+
* @return The OPDS feed.
77+
*/
78+
std::string languagesOPDSFeed() const;
7379

7480
/**
7581
* Set the id of the library.

‎src/library.cpp

+27-42
Original file line numberDiff line numberDiff line change
@@ -208,23 +208,36 @@ bool Library::writeBookmarksToFile(const std::string& path) const
208208
return writeTextFile(path, dumper.dumpLibXMLBookmark());
209209
}
210210

211-
std::vector<std::string> Library::getBooksLanguages() const
211+
Library::AttributeCounts Library::getBookAttributeCounts(BookStrPropMemFn p) const
212212
{
213-
std::vector<std::string> booksLanguages;
214-
std::map<std::string, bool> booksLanguagesMap;
213+
AttributeCounts propValueCounts;
215214

216-
for (auto& pair: m_books) {
217-
auto& book = pair.second;
218-
auto& language = book.getLanguage();
219-
if (booksLanguagesMap.find(language) == booksLanguagesMap.end()) {
220-
if (book.getOrigId().empty()) {
221-
booksLanguagesMap[language] = true;
222-
booksLanguages.push_back(language);
223-
}
215+
for (const auto& pair: m_books) {
216+
const auto& book = pair.second;
217+
if (book.getOrigId().empty()) {
218+
propValueCounts[(book.*p)()] += 1;
224219
}
225220
}
221+
return propValueCounts;
222+
}
223+
224+
std::vector<std::string> Library::getBookPropValueSet(BookStrPropMemFn p) const
225+
{
226+
std::vector<std::string> result;
227+
for ( const auto& kv : getBookAttributeCounts(p) ) {
228+
result.push_back(kv.first);
229+
}
230+
return result;
231+
}
232+
233+
std::vector<std::string> Library::getBooksLanguages() const
234+
{
235+
return getBookPropValueSet(&Book::getLanguage);
236+
}
226237

227-
return booksLanguages;
238+
Library::AttributeCounts Library::getBooksLanguagesWithCounts() const
239+
{
240+
return getBookAttributeCounts(&Book::getLanguage);
228241
}
229242

230243
std::vector<std::string> Library::getBooksCategories() const
@@ -244,40 +257,12 @@ std::vector<std::string> Library::getBooksCategories() const
244257

245258
std::vector<std::string> Library::getBooksCreators() const
246259
{
247-
std::vector<std::string> booksCreators;
248-
std::map<std::string, bool> booksCreatorsMap;
249-
250-
for (auto& pair: m_books) {
251-
auto& book = pair.second;
252-
auto& creator = book.getCreator();
253-
if (booksCreatorsMap.find(creator) == booksCreatorsMap.end()) {
254-
if (book.getOrigId().empty()) {
255-
booksCreatorsMap[creator] = true;
256-
booksCreators.push_back(creator);
257-
}
258-
}
259-
}
260-
261-
return booksCreators;
260+
return getBookPropValueSet(&Book::getCreator);
262261
}
263262

264263
std::vector<std::string> Library::getBooksPublishers() const
265264
{
266-
std::vector<std::string> booksPublishers;
267-
std::map<std::string, bool> booksPublishersMap;
268-
269-
for (auto& pair:m_books) {
270-
auto& book = pair.second;
271-
auto& publisher = book.getPublisher();
272-
if (booksPublishersMap.find(publisher) == booksPublishersMap.end()) {
273-
if (book.getOrigId().empty()) {
274-
booksPublishersMap[publisher] = true;
275-
booksPublishers.push_back(publisher);
276-
}
277-
}
278-
}
279-
280-
return booksPublishers;
265+
return getBookPropValueSet(&Book::getPublisher);
281266
}
282267

283268
const std::vector<kiwix::Bookmark> Library::getBookmarks(bool onlyValidBookmarks) const

‎src/opds_dumper.cpp

+40-2
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222

2323
#include "kiwixlib-resources.h"
2424
#include <mustache.hpp>
25+
#include <unicode/locid.h>
2526

2627
#include "tools/stringTools.h"
2728
#include "tools/otherTools.h"
@@ -83,6 +84,15 @@ BookData getBookData(const Library* library, const std::vector<std::string>& boo
8384
return bookData;
8485
}
8586

87+
std::string getLanguageSelfName(const std::string& lang) {
88+
const icu::Locale locale(lang.c_str());
89+
icu::UnicodeString ustring;
90+
locale.getDisplayLanguage(locale, ustring);
91+
std::string result;
92+
ustring.toUTF8String(result);
93+
return result;
94+
};
95+
8696
} // unnamed namespace
8797

8898
string OPDSDumper::dumpOPDSFeed(const std::vector<std::string>& bookIds, const std::string& query) const
@@ -121,11 +131,11 @@ string OPDSDumper::dumpOPDSFeedV2(const std::vector<std::string>& bookIds, const
121131
return render_template(RESOURCE::templates::catalog_v2_entries_xml, template_data);
122132
}
123133

124-
std::string OPDSDumper::categoriesOPDSFeed(const std::vector<std::string>& categories) const
134+
std::string OPDSDumper::categoriesOPDSFeed() const
125135
{
126136
const auto now = gen_date_str();
127137
kainjow::mustache::list categoryData;
128-
for ( const auto& category : categories ) {
138+
for ( const auto& category : library->getBooksCategories() ) {
129139
const auto urlencodedCategoryName = urlEncode(category);
130140
categoryData.push_back(kainjow::mustache::object{
131141
{"name", category},
@@ -146,4 +156,32 @@ std::string OPDSDumper::categoriesOPDSFeed(const std::vector<std::string>& categ
146156
);
147157
}
148158

159+
std::string OPDSDumper::languagesOPDSFeed() const
160+
{
161+
const auto now = gen_date_str();
162+
kainjow::mustache::list languageData;
163+
for ( const auto& langAndBookCount : library->getBooksLanguagesWithCounts() ) {
164+
const std::string languageCode = langAndBookCount.first;
165+
const int bookCount = langAndBookCount.second;
166+
const auto languageSelfName = getLanguageSelfName(languageCode);
167+
languageData.push_back(kainjow::mustache::object{
168+
{"lang_code", languageCode},
169+
{"lang_self_name", languageSelfName},
170+
{"book_count", to_string(bookCount)},
171+
{"updated", now},
172+
{"id", gen_uuid(libraryId + "/languages/" + languageCode)}
173+
});
174+
}
175+
176+
return render_template(
177+
RESOURCE::templates::catalog_v2_languages_xml,
178+
kainjow::mustache::object{
179+
{"date", now},
180+
{"endpoint_root", rootLocation + "/catalog/v2"},
181+
{"feed_id", gen_uuid(libraryId + "/languages")},
182+
{"languages", languageData }
183+
}
184+
);
185+
}
186+
149187
}

‎src/server/internalServer.h

+1
Original file line numberDiff line numberDiff line change
@@ -77,6 +77,7 @@ class InternalServer {
7777
std::unique_ptr<Response> handle_catalog_v2_root(const RequestContext& request);
7878
std::unique_ptr<Response> handle_catalog_v2_entries(const RequestContext& request);
7979
std::unique_ptr<Response> handle_catalog_v2_categories(const RequestContext& request);
80+
std::unique_ptr<Response> handle_catalog_v2_languages(const RequestContext& request);
8081
std::unique_ptr<Response> handle_meta(const RequestContext& request);
8182
std::unique_ptr<Response> handle_search(const RequestContext& request);
8283
std::unique_ptr<Response> handle_suggest(const RequestContext& request);

‎src/server/internalServer_catalog_v2.cpp

+17-2
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,8 @@ std::unique_ptr<Response> InternalServer::handle_catalog_v2(const RequestContext
5959
return handle_catalog_v2_entries(request);
6060
} else if (url == "categories") {
6161
return handle_catalog_v2_categories(request);
62+
} else if (url == "languages") {
63+
return handle_catalog_v2_languages(request);
6264
} else {
6365
return Response::build_404(*this, request, "", "");
6466
}
@@ -74,7 +76,8 @@ std::unique_ptr<Response> InternalServer::handle_catalog_v2_root(const RequestCo
7476
{"endpoint_root", m_root + "/catalog/v2"},
7577
{"feed_id", gen_uuid(m_library_id)},
7678
{"all_entries_feed_id", gen_uuid(m_library_id + "/entries")},
77-
{"category_list_feed_id", gen_uuid(m_library_id + "/categories")}
79+
{"category_list_feed_id", gen_uuid(m_library_id + "/categories")},
80+
{"language_list_feed_id", gen_uuid(m_library_id + "/languages")}
7881
},
7982
"application/atom+xml;profile=opds-catalog;kind=navigation"
8083
);
@@ -101,7 +104,19 @@ std::unique_ptr<Response> InternalServer::handle_catalog_v2_categories(const Req
101104
opdsDumper.setLibraryId(m_library_id);
102105
return ContentResponse::build(
103106
*this,
104-
opdsDumper.categoriesOPDSFeed(mp_library->getBooksCategories()),
107+
opdsDumper.categoriesOPDSFeed(),
108+
"application/atom+xml;profile=opds-catalog;kind=navigation"
109+
);
110+
}
111+
112+
std::unique_ptr<Response> InternalServer::handle_catalog_v2_languages(const RequestContext& request)
113+
{
114+
OPDSDumper opdsDumper(mp_library);
115+
opdsDumper.setRootLocation(m_root);
116+
opdsDumper.setLibraryId(m_library_id);
117+
return ContentResponse::build(
118+
*this,
119+
opdsDumper.languagesOPDSFeed(),
105120
"application/atom+xml;profile=opds-catalog;kind=navigation"
106121
);
107122
}

‎static/resources_list.txt

+1
Original file line numberDiff line numberDiff line change
@@ -48,5 +48,6 @@ templates/catalog_entries.xml
4848
templates/catalog_v2_root.xml
4949
templates/catalog_v2_entries.xml
5050
templates/catalog_v2_categories.xml
51+
templates/catalog_v2_languages.xml
5152
opensearchdescription.xml
5253
catalog_v2_searchdescription.xml

‎static/templates/catalog_v2_languages.xml

+27
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<feed xmlns="http://www.w3.org/2005/Atom"
3+
xmlns:dc="http://purl.org/dc/terms/"
4+
xmlns:opds="https://specs.opds.io/opds-1.2">
5+
<id>{{feed_id}}</id>
6+
<link rel="self"
7+
href="{{endpoint_root}}/languages"
8+
type="application/atom+xml;profile=opds-catalog;kind=navigation"/>
9+
<link rel="start"
10+
href="{{endpoint_root}}/root.xml"
11+
type="application/atom+xml;profile=opds-catalog;kind=navigation"/>
12+
<title>List of languages</title>
13+
<updated>{{date}}</updated>
14+
15+
{{#languages}}
16+
<entry>
17+
<title>{{lang_self_name}}</title>
18+
<dc:language>{{{lang_code}}}</dc:language>
19+
<thr:count>{{book_count}}</thr:count>
20+
<link rel="subsection"
21+
href="{{endpoint_root}}/entries?lang={{{lang_code}}}"
22+
type="application/atom+xml;profile=opds-catalog;kind=acquisition"/>
23+
<updated>{{updated}}</updated>
24+
<id>{{id}}</id>
25+
</entry>
26+
{{/languages}}
27+
</feed>

‎static/templates/catalog_v2_root.xml

+9
Original file line numberDiff line numberDiff line change
@@ -32,4 +32,13 @@
3232
<id>{{category_list_feed_id}}</id>
3333
<content type="text">List of all categories in this catalog.</content>
3434
</entry>
35+
<entry>
36+
<title>List of languages</title>
37+
<link rel="subsection"
38+
href="{{endpoint_root}}/languages"
39+
type="application/atom+xml;profile=opds-catalog;kind=navigation"/>
40+
<updated>{{date}}</updated>
41+
<id>{{language_list_feed_id}}</id>
42+
<content type="text">List of all languages in this catalog.</content>
43+
</entry>
3544
</feed>

‎test/data/library.xml

+4-4
Original file line numberDiff line numberDiff line change
@@ -21,11 +21,11 @@
2121
url="https://github.com/kiwix/libkiwix/raw/master/test/data/zimfile.zim"
2222
title="Ray (uncategorized) Charles"
2323
description="No category is assigned to this library entry."
24-
language="eng"
24+
language="rus"
2525
creator="Wikipedia"
2626
publisher="Kiwix"
2727
date="2020-03-31"
28-
name="wikipedia_en_ray_charles"
28+
name="wikipedia_ru_ray_charles"
2929
tags="unittest;wikipedia;_pictures:no;_videos:no;_details:no"
3030
articleCount="284"
3131
mediaCount="2"
@@ -37,11 +37,11 @@
3737
url="https://github.com/kiwix/libkiwix/raw/master/test/data/zimfile.zim"
3838
title="Charles, Ray"
3939
description="Wikipedia articles about Ray Charles"
40-
language="eng"
40+
language="fra"
4141
creator="Wikipedia"
4242
publisher="Kiwix"
4343
date="2020-03-31"
44-
name="wikipedia_en_ray_charles"
44+
name="wikipedia_fr_ray_charles"
4545
tags="unittest;wikipedia;_category:jazz;_pictures:no;_videos:no;_details:no;_ftindex:yes"
4646
articleCount="284"
4747
mediaCount="2"

‎test/library.cpp

+19-3
Original file line numberDiff line numberDiff line change
@@ -275,9 +275,25 @@ TEST_F(LibraryTest, getBookMarksTest)
275275
TEST_F(LibraryTest, sanityCheck)
276276
{
277277
EXPECT_EQ(lib.getBookCount(true, true), 12U);
278-
EXPECT_EQ(lib.getBooksLanguages().size(), 3U);
279-
EXPECT_EQ(lib.getBooksCreators().size(), 9U);
280-
EXPECT_EQ(lib.getBooksPublishers().size(), 3U);
278+
EXPECT_EQ(lib.getBooksLanguages(),
279+
std::vector<std::string>({"deu", "eng", "fra"})
280+
);
281+
EXPECT_EQ(lib.getBooksCreators(), std::vector<std::string>({
282+
"Islam Stack Exchange",
283+
"Movies & TV Stack Exchange",
284+
"Mythology & Folklore Stack Exchange",
285+
"TED",
286+
"Tania Louis",
287+
"Wiki",
288+
"Wikibooks",
289+
"Wikipedia",
290+
"Wikiquote"
291+
}));
292+
EXPECT_EQ(lib.getBooksPublishers(), std::vector<std::string>({
293+
"",
294+
"Kiwix",
295+
"Kiwix & Some Enthusiasts"
296+
}));
281297
}
282298

283299
TEST_F(LibraryTest, categoryHandling)

0 commit comments

Comments
 (0)