A fork of https://github.com/crosspoint-reader/crosspoint-reader
fix: OPDS browser OOM (#403)
## Summary
- Rewrite OpdsParser to stream parsing instead of full content
- Fix OOM due to big http xml response
Closes #385
---
### AI Usage
While CrossPoint doesn't have restrictions on AI tools in contributing,
please be transparent about their usage as it
helps set the right context for reviewers.
Did you use AI tools to help write this code? _**NO**_
authored by
KasyanDiGris
and committed by
GitHub
47ef92e8
e3d6e326
+111
-39
+28
-17
lib/OpdsParser/OpdsParser.cpp
+28
-17
lib/OpdsParser/OpdsParser.cpp
···
4
4
5
5
#include <cstring>
6
6
7
+
OpdsParser::OpdsParser() {
8
+
parser = XML_ParserCreate(nullptr);
9
+
if (!parser) {
10
+
errorOccured = true;
11
+
Serial.printf("[%lu] [OPDS] Couldn't allocate memory for parser\n", millis());
12
+
}
13
+
}
14
+
7
15
OpdsParser::~OpdsParser() {
8
16
if (parser) {
9
17
XML_StopParser(parser, XML_FALSE);
···
14
22
}
15
23
}
16
24
17
-
bool OpdsParser::parse(const char* xmlData, const size_t length) {
18
-
clear();
25
+
size_t OpdsParser::write(uint8_t c) { return write(&c, 1); }
19
26
20
-
parser = XML_ParserCreate(nullptr);
21
-
if (!parser) {
22
-
Serial.printf("[%lu] [OPDS] Couldn't allocate memory for parser\n", millis());
23
-
return false;
27
+
size_t OpdsParser::write(const uint8_t* xmlData, const size_t length) {
28
+
if (errorOccured) {
29
+
return length;
24
30
}
25
31
26
32
XML_SetUserData(parser, this);
···
28
34
XML_SetCharacterDataHandler(parser, characterData);
29
35
30
36
// Parse in chunks to avoid large buffer allocations
31
-
const char* currentPos = xmlData;
37
+
const char* currentPos = reinterpret_cast<const char*>(xmlData);
32
38
size_t remaining = length;
33
39
constexpr size_t chunkSize = 1024;
34
40
35
41
while (remaining > 0) {
36
42
void* const buf = XML_GetBuffer(parser, chunkSize);
37
43
if (!buf) {
44
+
errorOccured = true;
38
45
Serial.printf("[%lu] [OPDS] Couldn't allocate memory for buffer\n", millis());
39
46
XML_ParserFree(parser);
40
47
parser = nullptr;
41
-
return false;
48
+
return length;
42
49
}
43
50
44
51
const size_t toRead = remaining < chunkSize ? remaining : chunkSize;
45
52
memcpy(buf, currentPos, toRead);
46
53
47
-
const bool isFinal = (remaining == toRead);
48
-
if (XML_ParseBuffer(parser, static_cast<int>(toRead), isFinal) == XML_STATUS_ERROR) {
54
+
if (XML_ParseBuffer(parser, static_cast<int>(toRead), 0) == XML_STATUS_ERROR) {
55
+
errorOccured = true;
49
56
Serial.printf("[%lu] [OPDS] Parse error at line %lu: %s\n", millis(), XML_GetCurrentLineNumber(parser),
50
57
XML_ErrorString(XML_GetErrorCode(parser)));
51
58
XML_ParserFree(parser);
52
59
parser = nullptr;
53
-
return false;
60
+
return length;
54
61
}
55
62
56
63
currentPos += toRead;
57
64
remaining -= toRead;
58
65
}
59
-
60
-
// Clean up parser
61
-
XML_ParserFree(parser);
62
-
parser = nullptr;
66
+
return length;
67
+
}
63
68
64
-
Serial.printf("[%lu] [OPDS] Parsed %zu entries\n", millis(), entries.size());
65
-
return true;
69
+
void OpdsParser::flush() {
70
+
if (XML_Parse(parser, nullptr, 0, XML_TRUE) != XML_STATUS_OK) {
71
+
errorOccured = true;
72
+
XML_ParserFree(parser);
73
+
parser = nullptr;
74
+
}
66
75
}
76
+
77
+
bool OpdsParser::error() const { return errorOccured; }
67
78
68
79
void OpdsParser::clear() {
69
80
entries.clear();
+15
-10
lib/OpdsParser/OpdsParser.h
+15
-10
lib/OpdsParser/OpdsParser.h
···
1
1
#pragma once
2
+
#include <Print.h>
2
3
#include <expat.h>
3
4
4
5
#include <string>
···
42
43
* }
43
44
* }
44
45
*/
45
-
class OpdsParser {
46
+
class OpdsParser final : public Print {
46
47
public:
47
-
OpdsParser() = default;
48
+
OpdsParser();
48
49
~OpdsParser();
49
50
50
51
// Disable copy
51
52
OpdsParser(const OpdsParser&) = delete;
52
53
OpdsParser& operator=(const OpdsParser&) = delete;
53
54
54
-
/**
55
-
* Parse an OPDS XML feed.
56
-
* @param xmlData Pointer to the XML data
57
-
* @param length Length of the XML data
58
-
* @return true if parsing succeeded, false on error
59
-
*/
60
-
bool parse(const char* xmlData, size_t length);
55
+
size_t write(uint8_t) override;
56
+
size_t write(const uint8_t*, size_t) override;
57
+
58
+
void flush() override;
59
+
60
+
bool error() const;
61
+
62
+
operator bool() { return !error(); }
61
63
62
64
/**
63
65
* Get the parsed entries (both navigation and book entries).
64
66
* @return Vector of OpdsEntry entries
65
67
*/
66
-
const std::vector<OpdsEntry>& getEntries() const { return entries; }
68
+
const std::vector<OpdsEntry>& getEntries() const& { return entries; }
69
+
std::vector<OpdsEntry> getEntries() && { return std::move(entries); }
67
70
68
71
/**
69
72
* Get only book entries (legacy compatibility).
···
96
99
bool inAuthor = false;
97
100
bool inAuthorName = false;
98
101
bool inId = false;
102
+
103
+
bool errorOccured = false;
99
104
};
+15
lib/OpdsParser/OpdsStream.cpp
+15
lib/OpdsParser/OpdsStream.cpp
···
1
+
#include "OpdsStream.h"
2
+
3
+
OpdsParserStream::OpdsParserStream(OpdsParser& parser) : parser(parser) {}
4
+
5
+
int OpdsParserStream::available() { return 0; }
6
+
7
+
int OpdsParserStream::peek() { abort(); }
8
+
9
+
int OpdsParserStream::read() { abort(); }
10
+
11
+
size_t OpdsParserStream::write(uint8_t c) { return parser.write(c); }
12
+
13
+
size_t OpdsParserStream::write(const uint8_t* buffer, size_t size) { return parser.write(buffer, size); }
14
+
15
+
OpdsParserStream::~OpdsParserStream() { parser.flush(); }
+23
lib/OpdsParser/OpdsStream.h
+23
lib/OpdsParser/OpdsStream.h
···
1
+
#pragma once
2
+
3
+
#include <Stream.h>
4
+
5
+
#include "OpdsParser.h"
6
+
7
+
class OpdsParserStream : public Stream {
8
+
public:
9
+
explicit OpdsParserStream(OpdsParser& parser);
10
+
11
+
// That functions are not implimented for that stream
12
+
int available() override;
13
+
int peek() override;
14
+
int read() override;
15
+
16
+
virtual size_t write(uint8_t c) override;
17
+
virtual size_t write(const uint8_t* buffer, size_t size) override;
18
+
19
+
~OpdsParserStream() override;
20
+
21
+
private:
22
+
OpdsParser& parser;
23
+
};
+14
-9
src/activities/browser/OpdsBookBrowserActivity.cpp
+14
-9
src/activities/browser/OpdsBookBrowserActivity.cpp
···
3
3
#include <Epub.h>
4
4
#include <GfxRenderer.h>
5
5
#include <HardwareSerial.h>
6
+
#include <OpdsStream.h>
6
7
#include <WiFi.h>
7
8
8
9
#include "CrossPointSettings.h"
···
265
266
std::string url = UrlUtils::buildUrl(serverUrl, path);
266
267
Serial.printf("[%lu] [OPDS] Fetching: %s\n", millis(), url.c_str());
267
268
268
-
std::string content;
269
-
if (!HttpDownloader::fetchUrl(url, content)) {
270
-
state = BrowserState::ERROR;
271
-
errorMessage = "Failed to fetch feed";
272
-
updateRequired = true;
273
-
return;
269
+
OpdsParser parser;
270
+
271
+
{
272
+
OpdsParserStream stream{parser};
273
+
if (!HttpDownloader::fetchUrl(url, stream)) {
274
+
state = BrowserState::ERROR;
275
+
errorMessage = "Failed to fetch feed";
276
+
updateRequired = true;
277
+
return;
278
+
}
274
279
}
275
280
276
-
OpdsParser parser;
277
-
if (!parser.parse(content.c_str(), content.size())) {
281
+
if (!parser) {
278
282
state = BrowserState::ERROR;
279
283
errorMessage = "Failed to parse feed";
280
284
updateRequired = true;
281
285
return;
282
286
}
283
287
284
-
entries = parser.getEntries();
288
+
entries = std::move(parser).getEntries();
289
+
Serial.printf("[%lu] [OPDS] Found %d entries\n", millis(), entries.size());
285
290
selectorIndex = 0;
286
291
287
292
if (entries.empty()) {
+14
-3
src/network/HttpDownloader.cpp
+14
-3
src/network/HttpDownloader.cpp
···
2
2
3
3
#include <HTTPClient.h>
4
4
#include <HardwareSerial.h>
5
+
#include <StreamString.h>
5
6
#include <WiFiClient.h>
6
7
#include <WiFiClientSecure.h>
7
8
···
9
10
10
11
#include "util/UrlUtils.h"
11
12
12
-
bool HttpDownloader::fetchUrl(const std::string& url, std::string& outContent) {
13
+
bool HttpDownloader::fetchUrl(const std::string& url, Stream& outContent) {
13
14
// Use WiFiClientSecure for HTTPS, regular WiFiClient for HTTP
14
15
std::unique_ptr<WiFiClient> client;
15
16
if (UrlUtils::isHttpsUrl(url)) {
···
34
35
return false;
35
36
}
36
37
37
-
outContent = http.getString().c_str();
38
+
http.writeToStream(&outContent);
39
+
38
40
http.end();
39
41
40
-
Serial.printf("[%lu] [HTTP] Fetched %zu bytes\n", millis(), outContent.size());
42
+
Serial.printf("[%lu] [HTTP] Fetch success\n", millis());
43
+
return true;
44
+
}
45
+
46
+
bool HttpDownloader::fetchUrl(const std::string& url, std::string& outContent) {
47
+
StreamString stream;
48
+
if (!fetchUrl(url, stream)) {
49
+
return false;
50
+
}
51
+
outContent = stream.c_str();
41
52
return true;
42
53
}
43
54