From dbeae862c5457eac7b1137fd84bd2f149518997f Mon Sep 17 00:00:00 2001 From: Mike Nolan Date: Tue, 22 Sep 2026 11:09:50 -0500 Subject: [PATCH] Fixed a stoll bug that was there since day one, html parser, fix pkgconfig --- .gitea/workflows/tag.yaml | 12 +- CMakeLists.txt | 1 - README.md | 1 + apps/thtml2json.cpp | 84 ++++ apps/topengraph.cpp | 77 ++++ apps/ttorrentcreate.cpp | 7 +- changelog.md | 3 + cmake/apps.cmake | 9 + cmake/findmbed.cmake | 14 +- cmake/sources.cmake | 1 + cmake/version.cmake | 2 +- include/TessesFramework/Http/HttpServer.hpp | 2 +- include/TessesFramework/Http/HttpUtils.hpp | 6 +- .../Serialization/BitConverter.hpp | 6 + .../TessesFramework/Serialization/Html.hpp | 71 +++ include/TessesFramework/TessesFramework.hpp | 1 + pkgconfig/CMakeLists.txt | 2 + src/Http/HttpServer.cpp | 138 ++++-- src/Http/HttpUtils.cpp | 173 ++++++- src/Serialization/BitConverter.cpp | 57 +++ src/Serialization/Html.cpp | 429 ++++++++++++++++++ 21 files changed, 1016 insertions(+), 80 deletions(-) create mode 100644 apps/thtml2json.cpp create mode 100644 apps/topengraph.cpp create mode 100644 include/TessesFramework/Serialization/Html.hpp create mode 100644 src/Serialization/Html.cpp diff --git a/.gitea/workflows/tag.yaml b/.gitea/workflows/tag.yaml index 9d46c95..f1bafe5 100644 --- a/.gitea/workflows/tag.yaml +++ b/.gitea/workflows/tag.yaml @@ -11,7 +11,7 @@ env: jobs: build-ppc: name: "🔨 Build for PowerPC" - runs-on: fw13-ubuntu-latest + runs-on: asrock container: image: git.tesses.org/tesses50/linux-ppc:latest steps: @@ -26,7 +26,7 @@ jobs: artifacts/** update-tap-and-build-win32: name: "🔨 Build win32 and update the tap 🍺" - runs-on: fw13-ubuntu-latest + runs-on: msi-walmart container: git.tesses.org/tesses50/global-containers-mingw:latest steps: - uses: actions/checkout@v4 @@ -58,27 +58,31 @@ jobs: git push build-packages: name: "🔨 Build for everything else" - runs-on: fw13-ubuntu-latest strategy: fail-fast: true matrix: include: - container: "git.tesses.org/tesses50/linux-x64:latest" + puter: "msi-walmart" nfpmName: "amd64" debianName: "x86_64-linux-gnu" - container: "git.tesses.org/tesses50/linux-x86:latest" + puter: "asrock" nfpmName: "386" debianName: "i386-linux-gnu" - container: "git.tesses.org/tesses50/linux-arm:latest" + puter: "fw13-ubuntu-latest" nfpmName: "arm7" debianName: "arm-linux-gnueabihf" - container: "git.tesses.org/tesses50/linux-arm64:latest" + puter: "msi-walmart" nfpmName: "arm64" debianName: "aarch64-linux-gnu" - container: "git.tesses.org/tesses50/linux-riscv64:latest" nfpmName: "riscv64" debianName: "riscv64-linux-gnu" - + puter: "asrock" + runs-on: ${{ matrix.puter }} container: image: ${{ matrix.container }} steps: diff --git a/CMakeLists.txt b/CMakeLists.txt index 6cc364c..3e27222 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -7,7 +7,6 @@ set(CMAKE_CXX_STANDARD 17) include(GNUInstallDirs) include(CheckLibraryExists) include(FetchContent) -include(cmake/sources.cmake) include(cmake/options.cmake) set(CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS ON) diff --git a/README.md b/README.md index 5b3c73f..20e2150 100644 --- a/README.md +++ b/README.md @@ -22,6 +22,7 @@ Tesses Framework - EventLoop - Basic crytography using mbedtls - a LCG Random number generator (do not use for crypto, use it for non security things like number guessing games and stuff, where rand would be ok) +- Simple HTML parser ## Supported Platforms - Linux diff --git a/apps/thtml2json.cpp b/apps/thtml2json.cpp new file mode 100644 index 0000000..a48cc24 --- /dev/null +++ b/apps/thtml2json.cpp @@ -0,0 +1,84 @@ +#include "TessesFramework/TessesFramework.hpp" +using namespace Tesses::Framework::Streams; +using namespace Tesses::Framework::Serialization; +using namespace Tesses::Framework::TextStreams; + +Json::JToken Html2JToken(std::shared_ptr node) { + if (node->isText) { + return node->text_or_tag; + } else { + Json::JObject obj; + obj.SetValue("Tag", node->text_or_tag); + Json::JArray attrs; + for (auto &item : node->attributes) { + if (item.second) + attrs.Add(Json::JObject( + {{"Key", item.first}, {"Value", item.second.value()}})); + else + attrs.Add( + Json::JObject({{"Key", item.first}, {"Value", nullptr}})); + } + obj.SetValue("Attributes", attrs); + + Json::JArray childern; + + for (auto &item : node->children) + childern.Add(Html2JToken(item)); + + obj.SetValue("Children", childern); + return obj; + } +} + +std::shared_ptr OpenWrite(std::string dest) { + if (dest == "-") { + return std::make_shared(stdout, false, "w"); + } else { + auto strm = std::make_shared(dest, "w"); + if (!strm->CanWrite()) { + return nullptr; + } + return strm; + } +} +std::shared_ptr OpenRead(std::string src) { + if (src == "-") { + return std::make_shared(stdin, false, "r"); + } else { + auto strm = std::make_shared(src, "r"); + if (!strm->CanRead()) { + return nullptr; + } + return strm; + } +} +int main(int argc, char **argv) { + if (argc < 3) { + std::cout << "USAGE: " << argv[0] << " SRC DEST" << std::endl; + std::cout << "SRC: html file or - for stdin to convert to json doc" + << std::endl; + std::cout << "DEST: json file or - for stdout" << std::endl; + return 0; + } + auto src = OpenRead(argv[1]); + + auto dest = OpenWrite(argv[2]); + + if (src == nullptr) { + + std::cerr << "ERROR: Input could not be read" << std::endl; + return 1; + } + if (dest == nullptr) { + std::cerr << "ERROR: Output could not be read" << std::endl; + return 1; + } + + StreamWriter writer(dest); + + auto html = Html::Parse(std::make_shared(src)); + + writer.WriteLine(Json::Json::Encode(Html2JToken(html))); + + return 0; +} \ No newline at end of file diff --git a/apps/topengraph.cpp b/apps/topengraph.cpp new file mode 100644 index 0000000..4d4fcfb --- /dev/null +++ b/apps/topengraph.cpp @@ -0,0 +1,77 @@ +#include "TessesFramework/TessesFramework.hpp" +using namespace Tesses::Framework::Streams; +using namespace Tesses::Framework::Serialization; +using namespace Tesses::Framework::TextStreams; + +std::shared_ptr OpenRead(std::string src) { + + if (src == "-") { + return std::make_shared(stdin, false, "r"); + } else if (src.find("http://") == 0 || src.find("https://") == 0) { + Tesses::Framework::Http::HttpRequest req; + req.url = src; + req.method = "GET"; + Tesses::Framework::Http::HttpResponse resp(req); + + if ((int)resp.statusCode < 200 || (int)resp.statusCode > 299) { + throw std::runtime_error( + "Response does not indicate success: " + + std::to_string((int)resp.statusCode) + " " + + Tesses::Framework::Http::HttpUtils::StatusCodeString( + resp.statusCode) + + "."); + } + return resp.ReadAsStream(); + } else { + auto strm = std::make_shared(src, "r"); + if (!strm->CanRead()) { + return nullptr; + } + return strm; + } +} +int main(int argc, char **argv) { + if (argc < 2) { + std::cout << "USAGE: " << argv[0] << " SRC" << std::endl; + std::cout << "SRC: html file or url or - for stdin" << std::endl; + return 0; + } + auto src = OpenRead(argv[1]); + + if (src == nullptr) { + + std::cerr << "ERROR: Input could not be read" << std::endl; + return 1; + } + + auto html = Html::Parse(std::make_shared(src)); + std::vector> titleTags; + html->FindNodes("title", titleTags); + + for (auto &title : titleTags) { + Tesses::Framework::Console::WriteView("Title: "); + Tesses::Framework::Console::WriteLine(title->InnerText()); + break; + } + + std::vector> metaTags; + html->FindNodes("meta", metaTags); + + for (auto &meta : metaTags) { + std::optional prop, content; + for (auto &[k, v] : meta->attributes) { + if (k == "property" && v) + prop = *v; + if (k == "content" && v) + content = *v; + } + + if (prop && content && prop->find("og:") == 0) { + Tesses::Framework::Console::WriteView(prop.value()); + Tesses::Framework::Console::WriteView(": "); + Tesses::Framework::Console::WriteLineView(content.value()); + } + } + + return 0; +} \ No newline at end of file diff --git a/apps/ttorrentcreate.cpp b/apps/ttorrentcreate.cpp index 9bb7d5b..7e9a528 100644 --- a/apps/ttorrentcreate.cpp +++ b/apps/ttorrentcreate.cpp @@ -39,7 +39,7 @@ int main(int argc, char **argv) { Args args(argc, argv); if (args.positional.size() < 2) usage(args); - + int64_t num; for (auto &opt : args.options) { if (opt.first == "tracker") trackers.push_back(opt.second); @@ -49,8 +49,9 @@ int main(int argc, char **argv) { comment = opt.second; else if (opt.first == "created_by") created_by = opt.second; - else if (opt.first == "piece_length") - pieceLength = std::stoll(opt.second); + else if (opt.first == "piece_length" && + Serialization::BitConverter::TryParseSigned(opt.second, num)) + pieceLength = num; } for (auto &flag : args.flags) { diff --git a/changelog.md b/changelog.md index 86deb40..0609286 100644 --- a/changelog.md +++ b/changelog.md @@ -1,5 +1,8 @@ # Changelog +## 0.0.8 +Fixed a stoll bug that was there since day one, html parser, fix pkgconfig + ## 0.0.7 Fix license compliance and fix things up, add reverse proxy, fix some security flaws with HttpUtils diff --git a/cmake/apps.cmake b/cmake/apps.cmake index 8db6abd..292eca9 100644 --- a/cmake/apps.cmake +++ b/cmake/apps.cmake @@ -73,4 +73,13 @@ add_executable(tsimplereverseproxy apps/tsimplereverseproxy.cpp) target_link_libraries(tsimplereverseproxy PUBLIC tessesframework) install(TARGETS tsimplereverseproxy DESTINATION "${CMAKE_INSTALL_BINDIR}") + +add_executable(thtml2json apps/thtml2json.cpp) +target_link_libraries(thtml2json PUBLIC tessesframework) +install(TARGETS thtml2json DESTINATION "${CMAKE_INSTALL_BINDIR}") + + +add_executable(topengraph apps/topengraph.cpp) +target_link_libraries(topengraph PUBLIC tessesframework) +install(TARGETS topengraph DESTINATION "${CMAKE_INSTALL_BINDIR}") endif() \ No newline at end of file diff --git a/cmake/findmbed.cmake b/cmake/findmbed.cmake index 76bd13b..54ba365 100644 --- a/cmake/findmbed.cmake +++ b/cmake/findmbed.cmake @@ -18,21 +18,19 @@ if(NOT mbedtls_POPULATED) set(BUILD_SHARED_LIBS OFF CACHE BOOL "Build static" FORCE) if(NOT TESSESFRAMEWORK_STATIC) set(CMAKE_POSITION_INDEPENDENT_CODE ON CACHE BOOL "Build PIC" FORCE) - set(CMAKE_SKIP_INSTALL_RULES ON) - set(DISABLE_PACKAGE_CONFIG_AND_INSTALL ON CACHE BOOL "" FORCE) endif() FetchContent_Populate(mbedtls) - if(NOT TESSESFRAMEWORK_STATIC) - install(CODE "set(CMAKE_INSTALL_LOCAL_ONLY TRUE)" ALL_COMPONENTS) - endif() - - add_subdirectory(${mbedtls_SOURCE_DIR} ${mbedtls_BINARY_DIR}) + + if(NOT TESSESFRAMEWORK_STATIC) - set(CMAKE_SKIP_INSTALL_RULES OFF) + add_subdirectory(${mbedtls_SOURCE_DIR} ${mbedtls_BINARY_DIR} EXCLUDE_FROM_ALL) + else() + add_subdirectory(${mbedtls_SOURCE_DIR} ${mbedtls_BINARY_DIR}) endif() + endif() else() if(TESSESFRAMEWORK_MBEDTLS_PORTLIB STREQUAL "") diff --git a/cmake/sources.cmake b/cmake/sources.cmake index 2c4b761..435791e 100644 --- a/cmake/sources.cmake +++ b/cmake/sources.cmake @@ -22,6 +22,7 @@ src/Serialization/Json.cpp src/Serialization/SQLite.cpp src/Serialization/BitConverter.cpp src/Serialization/Bencode.cpp +src/Serialization/Html.cpp src/Platform/Environment.cpp src/Platform/Process.cpp src/Streams/FileStream.cpp diff --git a/cmake/version.cmake b/cmake/version.cmake index 89fb0d2..1f385ec 100644 --- a/cmake/version.cmake +++ b/cmake/version.cmake @@ -1,3 +1,3 @@ set(TESSESFRAMEWORK_MAJOR_VERSION 0) set(TESSESFRAMEWORK_MINOR_VERSION 0) -set(TESSESFRAMEWORK_PATCH_VERSION 7) +set(TESSESFRAMEWORK_PATCH_VERSION 8) diff --git a/include/TessesFramework/Http/HttpServer.hpp b/include/TessesFramework/Http/HttpServer.hpp index 06212e7..590fa03 100644 --- a/include/TessesFramework/Http/HttpServer.hpp +++ b/include/TessesFramework/Http/HttpServer.hpp @@ -98,7 +98,7 @@ class ServerContext { void SendErrorPage(bool showPath); void SendNotFound(); void SendBadRequest(); - void SendException(std::exception &ex); + void SendException(const std::exception &ex); void SendServerSentEvents(std::shared_ptr sse); std::shared_ptr OpenResponseStream(); std::shared_ptr OpenRequestStream(); diff --git a/include/TessesFramework/Http/HttpUtils.hpp b/include/TessesFramework/Http/HttpUtils.hpp index 304b907..ae3f8cf 100644 --- a/include/TessesFramework/Http/HttpUtils.hpp +++ b/include/TessesFramework/Http/HttpUtils.hpp @@ -157,9 +157,9 @@ class Uri { std::string GetPathAndQuery(); uint16_t GetPort(); std::string HostPort(); - bool Relative(std::string url, Uri &uri); + bool Relative(std::string_view url, Uri &uri); std::string ToString(); - static bool TryParse(std::string url, Uri &uri); + static bool TryParse(std::string_view url, Uri &uri); std::string scheme; std::string host; uint16_t port; @@ -199,6 +199,8 @@ class HttpUtils { static std::string UrlPathEncode(std::string_view v, bool ignoreSpace = false); static std::string HtmlEncode(std::string_view v); + // not all escapes are supported + static std::string HtmlDecode(std::string_view v); static std::string HtmlP(std::string_view text); static void SplitString(std::vector &out, std::string_view text, std::string_view delimiter, diff --git a/include/TessesFramework/Serialization/BitConverter.hpp b/include/TessesFramework/Serialization/BitConverter.hpp index a5a70c7..f047ecf 100644 --- a/include/TessesFramework/Serialization/BitConverter.hpp +++ b/include/TessesFramework/Serialization/BitConverter.hpp @@ -83,6 +83,12 @@ class BitConverter { static void FromUuid(uint8_t &b, const Uuid &uuid); + static bool TryParseSigned(const std::string &str, int64_t &val, + int base = 10); + static bool TryParseUnsigned(const std::string &str, uint64_t &val, + int base = 10); + static bool TryParseDouble(const std::string &str, double &val); + static inline bool IsLittleEndian() { uint8_t a[2]; a[0] = 0x01; diff --git a/include/TessesFramework/Serialization/Html.hpp b/include/TessesFramework/Serialization/Html.hpp new file mode 100644 index 0000000..c20d857 --- /dev/null +++ b/include/TessesFramework/Serialization/Html.hpp @@ -0,0 +1,71 @@ +/* + TessesFramework a library to make C++ easier for me, used in CrossLang: + https://git.tesses.org/tesses50/crosslang + + Copyright (C) 2026 Mike Nolan + SPDX-License-Identifier: GPL-3.0-or-later WITH TessesFramework-Exception-1.0 + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . +*/ + +#pragma once +#include "../TextStreams/TextReader.hpp" +namespace Tesses::Framework::Serialization::Html { + +struct HtmlNode; +struct HtmlNode : public std::enable_shared_from_this { + + std::string text_or_tag; + std::vector>> attributes; + std::vector> children; + bool isText = false; + + void FindNodes(std::string_view tag, + std::vector> &nodes) const; + std::vector> + FindNodes(std::string_view tag) const; + + void ToString(std::string &str) const; + std::string ToString() const; + + void InnerText(std::string &str) const; + std::string InnerText() const; + static std::shared_ptr FromText(std::string_view text); + static std::shared_ptr + FromNode(std::string_view name, + std::initializer_list< + std::pair>> + attrs, + std::initializer_list> nodes); + static std::shared_ptr FromNode( + std::string_view name, + std::vector>> attrs, + std::vector> nodes); + + template + static std::shared_ptr + FromNode(std::string_view name, IttrAttrs attrBegin, IttrAttrs attrEnd, + IttrNodes nodesBegin, IttrNodes nodesEnd) { + auto node = std::make_shared(); + node->isText = false; + node->text_or_tag = std::string{name}; + node->attributes.insert(node->attributes.end(), attrBegin, attrEnd); + node->children.insert(node->children.end(), nodesBegin, nodesEnd); + return node; + } +}; + +std::shared_ptr +Parse(std::shared_ptr reader); +} // namespace Tesses::Framework::Serialization::Html diff --git a/include/TessesFramework/TessesFramework.hpp b/include/TessesFramework/TessesFramework.hpp index ae0280e..20e0552 100644 --- a/include/TessesFramework/TessesFramework.hpp +++ b/include/TessesFramework/TessesFramework.hpp @@ -50,6 +50,7 @@ #include "Random.hpp" #include "Serialization/Bencode.hpp" #include "Serialization/BitConverter.hpp" +#include "Serialization/Html.hpp" #include "Serialization/Json.hpp" #include "Serialization/SQLite.hpp" #include "Streams/BufferedStream.hpp" diff --git a/pkgconfig/CMakeLists.txt b/pkgconfig/CMakeLists.txt index 680f702..d68a20a 100644 --- a/pkgconfig/CMakeLists.txt +++ b/pkgconfig/CMakeLists.txt @@ -11,6 +11,8 @@ set(PKGCONFIG_DEPS "") endif() configure_file(tessesframework.pc.in tessesframework.pc @ONLY) +message(NOTICE "SrcFile ${CMAKE_CURRENT_BINARY_DIR}/tessesframework.pc") +message(NOTICE "Outdir ${CMAKE_INSTALL_LIBDIR}/pkgconfig") install(FILES ${CMAKE_CURRENT_BINARY_DIR}/tessesframework.pc DESTINATION ${CMAKE_INSTALL_LIBDIR}/pkgconfig) diff --git a/src/Http/HttpServer.cpp b/src/Http/HttpServer.cpp index dc98225..8a1ae3b 100644 --- a/src/Http/HttpServer.cpp +++ b/src/Http/HttpServer.cpp @@ -39,6 +39,8 @@ #include "TessesFramework/Filesystem/VFS.hpp" #include "TessesFramework/Filesystem/VFSFix.hpp" // clang-format on + +#include "TessesFramework/Serialization/BitConverter.hpp" using FileStream = Tesses::Framework::Streams::FileStream; using Stream = Tesses::Framework::Streams::Stream; using SeekOrigin = Tesses::Framework::Streams::SeekOrigin; @@ -607,12 +609,34 @@ uint16_t HttpServer::GetPort() { return server->GetPort(); return 0; } + std::shared_ptr ServerContext::OpenResponseStream() { if (sent) return nullptr; int64_t length = -1; - if (!this->responseHeaders.TryGetFirstInt("Content-Length", length)) - length = -1; + try { + if (!this->responseHeaders.TryGetFirstInt("Content-Length", length)) + length = -1; + } catch (std::invalid_argument &invalid) { + std::string error = + " Internal Server Error at " + + HttpUtils::HtmlEncode(this->originalPath) + + "

Internal Server Error at " + + HttpUtils::HtmlEncode(this->originalPath) + + "

what(): " + HttpUtils::HtmlEncode(invalid.what()) + + "

"; + this->responseHeaders.SetValue("Content-Length", + static_cast(error.size())); + this->statusCode = StatusCode::InternalServerError; + this->WriteHeaders(); + this->strm->WriteBlock(reinterpret_cast(error.data()), + error.size()); + return nullptr; + } if (this->version == "HTTP/1.1" && length == -1) this->responseHeaders.SetValue("Transfer-Encoding", "chunked"); @@ -648,6 +672,10 @@ void HttpServer::StartAccepting() { uint16_t port; auto sock = svr->GetStream(ip, port); + if (!TF_IsRunning()) // we need to die, that continue is not what we + // want if we must die from being done + break; + TF_LOG("New Host IP: " + ip + ":" + std::to_string(port)); if (sock == nullptr) { @@ -708,9 +736,14 @@ ServerContext::ServerContext(std::shared_ptr strm, bool debug) } std::shared_ptr ServerContext::GetStream() { return this->strm; } void ServerContext::SendBytes(std::vector buff) { - std::shared_ptr strm = std::make_shared(false); - strm->GetBuffer() = buff; - SendStream(strm); + + if (sent) + return; + this->responseHeaders.SetValue("Content-Length", + static_cast(buff.size())); + auto resp = OpenResponseStream(); + if (resp) + resp->WriteBlock(buff.data(), buff.size()); } ServerContext &ServerContext::WithLastModified(Date::DateTime dt) { this->responseHeaders.SetValue("Last-Modified", dt); @@ -718,11 +751,13 @@ ServerContext &ServerContext::WithLastModified(Date::DateTime dt) { } void ServerContext::SendText(std::string text) { - std::shared_ptr strm = std::make_shared(false); - - auto &buff = strm->GetBuffer(); - buff.insert(buff.end(), text.begin(), text.end()); - SendStream(strm); + if (sent) + return; + this->responseHeaders.SetValue("Content-Length", + static_cast(text.size())); + auto resp = OpenResponseStream(); + if (resp) + resp->WriteBlock(reinterpret_cast(text.data()), text.size()); } void ServerContext::SendErrorPage(bool showPath) { if (sent) @@ -792,22 +827,34 @@ void ServerContext::SendStream(std::shared_ptr strm) { int64_t begin = 0; int64_t end = -1; - if (dash.size() == 1 && - res[0].find_first_of('-') != std::string::npos) { - // NUMBER- - begin = std::stoll(dash[0]); - } else if (dash.size() == 2) { + if (dash.size() == 2) { // NUMBER-NUMBER // or //-NUMBER - - if (dash[0].size() > 0) { - // NUMBER-NUMBER - begin = std::stoll(dash[0]); - end = std::stoll(dash[1]); - } else { + if (dash[0].empty()) { //-NUMBER - end = std::stoll(dash[1]); + + if (!Serialization::BitConverter::TryParseSigned( + dash[1], end)) + throw std::runtime_error( + "Failed to parse the end of the Range header"); + } else if (dash[1].empty()) { + if (!Serialization::BitConverter::TryParseSigned( + dash[0], begin)) + throw std::runtime_error( + "Failed to parse the end of the Range header"); + } else { + // NUMBER-NUMBER + if (!Serialization::BitConverter::TryParseSigned( + dash[0], begin)) + throw std::runtime_error( + "Failed to parse the beginning of the Range " + "header"); + + if (!Serialization::BitConverter::TryParseSigned( + dash[1], end)) + throw std::runtime_error( + "Failed to parse the end of the Range header"); } } else { this->statusCode = BadRequest; @@ -886,6 +933,8 @@ void ServerContext::SendStream(std::shared_ptr strm) { } else { auto chunkedStream = this->OpenResponseStream(); + if (!chunkedStream) + throw std::runtime_error("Can't open response stream"); if (method != "HEAD") strm->CopyTo(chunkedStream); @@ -945,24 +994,28 @@ ServerContext &ServerContext::WithStatusCode(StatusCode code) { this->statusCode = code; return *this; } -void ServerContext::SendException(std::exception &ex) { - if (this->debug) { +void ServerContext::SendException(const std::exception &ex) { + try { + if (this->debug) { - this->WithMimeType("text/html") - .WithStatusCode(StatusCode::InternalServerError) - .SendText(" Internal Server Error at " + - HttpUtils::HtmlEncode(this->originalPath) + - "

Internal Server Error at " + - HttpUtils::HtmlEncode(this->originalPath) + - "

what(): " + HttpUtils::HtmlEncode(ex.what()) + - "

"); - } else { - this->WithStatusCode(StatusCode::InternalServerError) - .SendErrorPage(true); + this->WithMimeType("text/html") + .WithStatusCode(StatusCode::InternalServerError) + .SendText( + " Internal Server Error at " + + HttpUtils::HtmlEncode(this->originalPath) + + "

Internal Server Error at " + + HttpUtils::HtmlEncode(this->originalPath) + + "

what(): " + HttpUtils::HtmlEncode(ex.what()) + + "

"); + } else { + this->WithStatusCode(StatusCode::InternalServerError) + .SendErrorPage(true); + } + } catch (...) { } } @@ -1106,6 +1159,13 @@ void HttpServer::Process(std::shared_ptr strm, ctx.requestHeaders.TryGetFirst("Content-Type", type) && type == "application/x-www-form-urlencoded" && ctx.requestHeaders.TryGetFirstInt("Content-Length", length)) { + + if (length <= 0) { + ctx.statusCode = StatusCode::BadRequest; + ctx.SendErrorPage(true); + return; + } + size_t len = (size_t)length; std::vector buffer(len); len = bStrm->ReadBlock(buffer.data(), len); diff --git a/src/Http/HttpUtils.cpp b/src/Http/HttpUtils.cpp index ca16b6d..03c51b4 100644 --- a/src/Http/HttpUtils.cpp +++ b/src/Http/HttpUtils.cpp @@ -21,14 +21,19 @@ #include "TessesFramework/Http/HttpUtils.hpp" #include "TessesFramework/Filesystem/VFS.hpp" +#include "TessesFramework/Serialization/BitConverter.hpp" +#include "TessesFramework/Text/StringConverter.hpp" #include #include #include using VFSPath = Tesses::Framework::Filesystem::VFSPath; namespace Tesses::Framework::Http { -bool Uri::Relative(std::string url, Uri &uri) { - auto index = url.find_first_of("//"); +bool Uri::Relative(std::string_view url, Uri &uri) { + auto path = this->path; + if (path.empty()) + path = "/"; + auto index = url.find("//"); if (index != std::string::npos) { if (Uri::TryParse(url, uri)) { if (index == 0) @@ -63,7 +68,10 @@ bool Uri::Relative(std::string url, Uri &uri) { auto fourthPart = HttpUtils::SplitString(thirdPart[1], "?", 2); - VFSPath p = VFSPath(this->path, fourthPart[0]); + VFSPath p = + VFSPath(path, (path.back() == '/' || fourthPart[0].empty()) + ? fourthPart[0] + : "../" + fourthPart[0]); uri.path = p.CollapseRelativeParents().ToString(); // this should be safe if (fourthPart.size() == 2) { @@ -98,21 +106,33 @@ uint16_t Uri::GetPort() { return 69; return 0; } -bool Uri::TryParse(std::string url, Uri &uri) { +bool Uri::TryParse(std::string_view url, Uri &uri) { uri.scheme = ""; uri.port = 0; auto firstPart = HttpUtils::SplitString(url, "//", 2); if (firstPart.size() == 2) uri.scheme = firstPart[0]; - else if (firstPart.empty()) + else return false; - auto secondPart = HttpUtils::SplitString( - firstPart.size() == 2 ? firstPart[1] : firstPart[0], "/", 2); + auto secondPart = HttpUtils::SplitString(firstPart[1], "/", 2); if (secondPart.size() == 1) { uri.path = "/"; + auto hp = secondPart[0]; + auto fragPart = HttpUtils::SplitString(hp, "#", 2); + if (fragPart.size() == 2) { + uri.hash = fragPart[1]; + hp = fragPart[0]; + } + auto queryPart = HttpUtils::SplitString(hp, "?", 2); + if (queryPart.size() == 2) { + HttpUtils::QueryParamsDecode(uri.query, queryPart[1]); + hp = queryPart[0]; + } + secondPart[0] = hp; + } else if (secondPart.size() == 2) { auto thirdPart = HttpUtils::SplitString(secondPart[1], "#", 2); if (thirdPart.empty()) @@ -135,14 +155,25 @@ bool Uri::TryParse(std::string url, Uri &uri) { if (hostPortPart.empty()) return false; if (hostPortPart.size() == 2) { - uri.port = (uint16_t)std::stoul(hostPortPart[1]); + uint64_t portNum; + if (!Serialization::BitConverter::TryParseUnsigned(hostPortPart[1], + portNum)) + return false; + if (portNum > 65535) + return false; + uri.port = static_cast(portNum); } uri.host = hostPortPart[0]; return true; } Uri::Uri() : query(true) {} -std::string Uri::GetPathAndQuery() { return this->path + this->GetQuery(); } +std::string Uri::GetPathAndQuery() { + if (this->path.empty() || this->path.front() != '/') + return '/' + this->path + this->GetQuery(); + + return this->path + this->GetQuery(); +} std::string Uri::GetQuery() { if (this->query.kvp.empty()) return ""; @@ -532,9 +563,16 @@ std::string HttpUtils::HtmlP(std::string_view text) { auto flush = [&]() -> void { if (!builder.empty()) { + // Collect trailing punctuation + std::string trailing; + static const char *punct = ".,;:!?'\""; + while (!builder.empty() && std::strchr(punct, builder.back())) { + trailing.insert(trailing.begin(), builder.back()); + builder.pop_back(); + } if (builder.find("http://") == 0 || builder.find("https://") == 0 || builder.find("ftp://") == 0 || builder.find("ftps://") == 0 || - builder.find("magnet:") == 0 || builder.find("btmh:") == 0) { + builder.find("magnet:") == 0) { newText += "" + HttpUtils::HtmlEncode(builder) + ""; } else if (builder.find("mailto:") == 0) { @@ -548,6 +586,7 @@ std::string HttpUtils::HtmlP(std::string_view text) { } else { newText += HttpUtils::HtmlEncode(builder); } + newText += HttpUtils::HtmlEncode(trailing); builder = ""; } }; @@ -564,7 +603,7 @@ std::string HttpUtils::HtmlP(std::string_view text) { break; case '\t': flush(); - newText += "&tab;"; + newText += "    "; break; case '\r': flush(); @@ -578,6 +617,106 @@ std::string HttpUtils::HtmlP(std::string_view text) { return newText; } +static std::unordered_map htmlencodings = { + {""", "\""}, + {"'", "'"}, + {"&", "&"}, + {"<", "<"}, + {">", ">"}, + {" ", "\xC2\xA0"}, + {"©", "\xC2\xA9"}, + {"®", "\xC2\xAE"}, + {"™", "\xE2\x84\xA2"}, + {"—", "\xE2\x80\x94"}, + {"–", "\xE2\x80\x93"}, + {"…", "\xE2\x80\xA6"}, + {"’", "\xE2\x80\x99"}, + {"‘", "\xE2\x80\x98"}, + {"”", "\xE2\x80\x9D"}, + {"“", "\xE2\x80\x9C"}, + {"€", "\xE2\x82\xAC"}, + {"£", "\xC2\xA3"}, + {"¥", "\xC2\xA5"}, + {"°", "\xC2\xB0"}, + {"±", "\xC2\xB1"}, + {"×", "\xC3\x97"}, + {"÷", "\xC3\xB7"}, + {"«", "\xC2\xAB"}, + {"»", "\xC2\xBB"}, + {"•", "\xE2\x80\xA2"}, + {"·", "\xC2\xB7"}, + {"½", "\xC2\xBD"}, + {"¼", "\xC2\xBC"}, + {"¾", "\xC2\xBE"}, +}; +std::string HttpUtils::HtmlDecode(std::string_view view) { + std::string myHtml = {}; + std::string tmp = {}; + bool inEscaped = false; + for (size_t i = 0; i < view.size(); i++) { + if (inEscaped) { + if (view[i] == ';') { + inEscaped = false; + + if (tmp.size() > 2 && tmp[1] == '#') //&#N + { + if (tmp[2] == 'x' || tmp[2] == 'X') { + auto res = tmp.substr(3); + uint64_t code; + if (Serialization::BitConverter::TryParseUnsigned( + res, code, 16)) { + if (code <= 0x10FFFF && + (code < 0xD800 || code > 0xDFFF)) { + Text::StringConverter::UTF8::FromUTF32( + myHtml, {static_cast(code)}); + } else { + myHtml += "\xEF\xBF\xBD"; + } + } else { + myHtml += "\xEF\xBF\xBD"; + } + } else { + auto res = tmp.substr(2); + uint64_t code; + if (Serialization::BitConverter::TryParseUnsigned( + res, code, 10)) { + if (code <= 0x10FFFF && + (code < 0xD800 || code > 0xDFFF)) { + Text::StringConverter::UTF8::FromUTF32( + myHtml, {static_cast(code)}); + } else { + myHtml += "\xEF\xBF\xBD"; + } + } else { + myHtml += "\xEF\xBF\xBD"; + } + } + } else { + tmp += ';'; + auto result = htmlencodings.find(tmp); + if (result != htmlencodings.end()) { + myHtml += result->second; + } else { + myHtml += "\xEF\xBF\xBD"; + } + } + } else { + tmp += view[i]; + } + } else { + if (view[i] == '&') { + tmp = "&"; + inEscaped = true; + } else { + myHtml += view[i]; + } + } + } + if (inEscaped) + myHtml += tmp; + return myHtml; +} + std::string HttpUtils::HtmlEncode(std::string_view html) { std::string myHtml = {}; for (auto item : html) { @@ -838,16 +977,8 @@ bool HttpDictionary::TryGetFirstInt(std::string key, int64_t &value) { std::string val; if (!TryGetFirst(key, val)) return false; - try { - size_t off = 0; - auto v = std::stoll(val, &off); - if (off != val.size()) - return false; - value = v; - } catch (std::exception &ex) { - return false; - } - return true; + + return Serialization::BitConverter::TryParseSigned(val, value); } bool HttpDictionary::TryGetFirstDate(std::string key, Date::DateTime &dt) { std::string val; diff --git a/src/Serialization/BitConverter.cpp b/src/Serialization/BitConverter.cpp index 7296357..d7ac6b8 100644 --- a/src/Serialization/BitConverter.cpp +++ b/src/Serialization/BitConverter.cpp @@ -284,4 +284,61 @@ void BitConverter::FromSint16LE(uint8_t &b, int16_t v) { memcpy(&dest, &v, sizeof(uint16_t)); FromUint16LE(b, dest); } + +bool BitConverter::TryParseSigned(const std::string &str, int64_t &val, + int base) { + try { + if (str.find_first_not_of(" \t") != 0) + return false; + size_t pos = 0; + int64_t v = std::stoll(str, &pos, base); + if (pos < str.size()) + return false; + val = v; + return true; + } catch (std::out_of_range &arg) { + return false; + } catch (std::invalid_argument &arg) { + return false; + } +} +bool BitConverter::TryParseUnsigned(const std::string &str, uint64_t &val, + int base) { + try { + + if (str.find_first_not_of(" \t") != 0) + return false; + if (str[0] == '-' || str[0] == '+') + return false; + size_t pos = 0; + uint64_t v = std::stoull(str, &pos, base); + if (pos < str.size()) + return false; + val = v; + return true; + } catch (std::out_of_range &arg) { + return false; + } catch (std::invalid_argument &arg) { + return false; + } +} +bool BitConverter::TryParseDouble(const std::string &str, double &val) { + try { + + if (str.find_first_not_of(" \t") != 0) + return false; + + size_t pos = 0; + double v = std::stod(str, &pos); + if (pos < str.size()) + return false; + val = v; + return true; + } catch (std::out_of_range &arg) { + return false; + } catch (std::invalid_argument &arg) { + return false; + } +} + } // namespace Tesses::Framework::Serialization \ No newline at end of file diff --git a/src/Serialization/Html.cpp b/src/Serialization/Html.cpp new file mode 100644 index 0000000..5cc7247 --- /dev/null +++ b/src/Serialization/Html.cpp @@ -0,0 +1,429 @@ +#include "TessesFramework/Serialization/Html.hpp" +#include "TessesFramework/Console.hpp" +#include "TessesFramework/Http/HttpUtils.hpp" +#include +namespace Tesses::Framework::Serialization::Html { + +static std::unordered_set html_nodes_with_no_body = { + "area", "base", "br", "col", "embed", "hr", "img", + "input", "link", "meta", "source", "track", "wbr", +}; +std::string HtmlNode::ToString() const { + std::string text; + HtmlNode::ToString(text); + return text; +} + +void HtmlNode::InnerText(std::string &str) const { + if (this->isText) { + str += this->text_or_tag; + } else { + for (const auto &item : this->children) + item->InnerText(str); + } +} + +std::string HtmlNode::InnerText() const { + std::string innerText; + InnerText(innerText); + return innerText; +} + +void HtmlNode::ToString(std::string &str) const { + if (this->isText) { + str += Http::HttpUtils::HtmlEncode(this->text_or_tag); + } else { + str += "<"; + str += this->text_or_tag; + for (auto &attr : this->attributes) { + str += ' '; + str += attr.first; + if (attr.second) { + str += "=\""; + str += Http::HttpUtils::HtmlEncode(attr.second.value()); + str += "\""; + } + } + str += ">"; + + if (html_nodes_with_no_body.count(this->text_or_tag) == 0) { + for (auto &child : this->children) + child->ToString(str); + + str += "text_or_tag; + str += ">"; + } + } +} +std::shared_ptr HtmlNode::FromText(std::string_view text) { + auto node = std::make_shared(); + node->isText = true; + node->text_or_tag = std::string{text}; + return node; +} +std::shared_ptr HtmlNode::FromNode( + std::string_view name, + std::initializer_list>> + attrs, + std::initializer_list> nodes) { + return HtmlNode::FromNode(name, attrs.begin(), attrs.end(), nodes.begin(), + nodes.end()); +} + +std::shared_ptr HtmlNode::FromNode( + std::string_view name, + std::vector>> attrs, + std::vector> nodes) { + return HtmlNode::FromNode(name, attrs.begin(), attrs.end(), nodes.begin(), + nodes.end()); +} + +void HtmlNode::FindNodes( + std::string_view tag, + std::vector> &nodes) const { + if (!this->isText) { + if (this->text_or_tag == tag) + nodes.push_back(this->shared_from_this()); + for (const auto &node : this->children) + node->FindNodes(tag, nodes); + } +} + +std::vector> +HtmlNode::FindNodes(std::string_view tag) const { + std::vector> nodes; + FindNodes(tag, nodes); + return nodes; +} + +enum class HtmlLexTokenType { SYMBOL, TEXT, STRING }; +struct HtmlLexToken { + std::string text; + HtmlLexTokenType type; + HtmlLexToken(HtmlLexTokenType type, std::string_view text) + : text(text), type(type) {} +}; + +struct HtmlLexer { + std::shared_ptr reader; + std::vector &tokens; + int32_t _peeked = -1; + std::string builder; + bool ignoreSpaceMode = false; + std::string nodeName = {}; + + HtmlLexer(std::shared_ptr reader, + std::vector &tokens) + : reader(std::move(reader)), tokens(tokens) {} + + int32_t Peek() { + if (_peeked != -1) + return _peeked; + _peeked = Read(); + return _peeked; + } + + int64_t Read() { + if (_peeked != -1) { + auto tmp = _peeked; + _peeked = -1; + return tmp; + } + + return reader->ReadChar(); + } + + void Flush() { + if (!builder.empty()) { + this->tokens.emplace_back(HtmlLexTokenType::TEXT, builder); + + if (ignoreSpaceMode && nodeName.empty()) + nodeName = builder; + + builder = ""; + } + } + + void SkipTo(std::string_view fmt) { + if (fmt.empty()) + return; + std::string window; + window.resize(fmt.size()); + + while (fmt != window) { + int32_t read = Read(); + if (read == -1) + break; + for (size_t i = 0; i < window.size() - 1; i++) { + window[i] = window[i + 1]; + } + + window.back() = static_cast(read); + } + } + + void Lex() { + int read; + while ((read = Read()) != -1) { + int peek = Peek(); + switch (read) { + case '"': + case '\'': { + if (ignoreSpaceMode) { + Flush(); + char quote = static_cast(read); + std::string val; + int c; + while ((c = Read()) != -1 && c != quote) + val += static_cast(c); + tokens.emplace_back(HtmlLexTokenType::STRING, val); + } else { + builder += static_cast(read); + } + break; + } + case '/': { + if (ignoreSpaceMode && peek == '>') { + Flush(); + Read(); + ignoreSpaceMode = false; + tokens.emplace_back(HtmlLexTokenType::SYMBOL, "/>"); + } else if (ignoreSpaceMode) { + builder += '/'; + } else { + builder += '/'; + } + break; + } + case '>': { + if (ignoreSpaceMode) { + Flush(); + ignoreSpaceMode = false; + tokens.emplace_back(HtmlLexTokenType::SYMBOL, ">"); + + if (nodeName == "script" || nodeName == "style") { + nodeName = "_"; + std::string builder2 = ""; + bool eof = false; + while ((read = Read()) != -1) { + if (read == '<') { + peek = Peek(); + if (peek == '/') { + Read(); + break; + } else if (peek == -1) { + eof = true; + break; + } else { + builder2 += '<'; + } + } else { + builder2 += static_cast(read); + } + } + + if (read != -1 && !eof) { + tokens.emplace_back(HtmlLexTokenType::TEXT, + builder2); + + ignoreSpaceMode = true; + tokens.emplace_back(HtmlLexTokenType::SYMBOL, "'; + } + } break; + case '<': { + Flush(); + if (peek == '!') { + // + // + Read(); + peek = Peek(); + if (peek == '-') { + Read(); + peek = Peek(); + if (peek == '-') { + Read(); + SkipTo("-->"); + + } else { + SkipTo(">"); + } + } else { + SkipTo(">"); + } + + } else if (peek == '/') { + Read(); + + ignoreSpaceMode = true; + tokens.emplace_back(HtmlLexTokenType::SYMBOL, "(read); + } else { + Flush(); + } + break; + default: + builder += static_cast(read); + break; + } + } + Flush(); + } +}; + +struct HtmlParser { + + std::vector &tokens; + size_t index = 0; + HtmlParser(std::vector &tokens) : tokens(tokens) {} + + bool IsSymbol(std::string_view type) { + if (Done()) + return false; + auto &obj = Peak(); + if (obj.type != HtmlLexTokenType::SYMBOL) + return false; + + if (obj.text != type) + return false; + index++; + return true; + } + + bool Done() { return index >= tokens.size(); } + + HtmlLexToken &Read() { return tokens[index++]; } + HtmlLexToken &Peak() { return tokens[index]; } + + void Parse(std::string_view name, std::shared_ptr &node, + bool root = false) { + while (!Done()) { + auto &curToken = Peak(); + switch (curToken.type) { + case HtmlLexTokenType::TEXT: { + Read(); + node->children.push_back(HtmlNode::FromText( + Http::HttpUtils::HtmlDecode(curToken.text))); + } break; + case HtmlLexTokenType::SYMBOL: { + if (curToken.text == "<") { + Read(); + if (Done()) { + return; + } + auto &val = Peak(); + if (val.type == HtmlLexTokenType::TEXT) { + Read(); + + std::shared_ptr newNode = + std::make_shared(); + newNode->isText = false; + newNode->text_or_tag = val.text; + + while (!Done()) { + auto &k = Peak(); + + if (k.type == HtmlLexTokenType::TEXT) { + Read(); + if (IsSymbol("=")) { + if (!Done()) { + auto &v = Peak(); + + if (v.type == + HtmlLexTokenType::STRING) { + Read(); + newNode->attributes.emplace_back( + k.text, + Http::HttpUtils::HtmlDecode( + v.text)); + continue; + } + } + } + newNode->attributes.emplace_back(k.text, + std::nullopt); + + continue; + } + + if (k.type == HtmlLexTokenType::SYMBOL && + (k.text == ">" || k.text == "/>")) + break; + } + if (!IsSymbol("/>")) { + IsSymbol(">"); + if (html_nodes_with_no_body.count(val.text) == 0) + Parse(val.text, newNode); + } + + node->children.push_back(newNode); + } + } else if (curToken.text == "children.push_back(HtmlNode::FromText(""); + + if (val.text == name && !root) + return; + } + } break; + } + } + } + + std::shared_ptr Parse() { + std::shared_ptr node = std::make_shared(); + node->isText = false; + node->text_or_tag = ":root"; + Parse(":root", node, true); + return node; + } +}; + +std::shared_ptr +Parse(std::shared_ptr reader) { + std::vector tokens; + + HtmlLexer lexer(reader, tokens); + + lexer.Lex(); + + HtmlParser parser(tokens); + + return parser.Parse(); +} + +} // namespace Tesses::Framework::Serialization::Html \ No newline at end of file