2 Commits
21 changed files with 1019 additions and 80 deletions
+11 -4
View File
@@ -11,11 +11,12 @@ env:
jobs:
build-ppc:
name: "🔨 Build for PowerPC"
runs-on: fw13-ubuntu-latest
runs-on: asrock
container:
image: git.tesses.org/tesses50/linux-ppc:latest
steps:
- uses: actions/checkout@v4
url: https://git.tesses.org/tesses50/tessesframework.git
- run: bash build-for-ppc.sh
- uses: akkuman/gitea-release-action@v1
env:
@@ -26,10 +27,11 @@ jobs:
artifacts/**
update-tap-and-build-win32:
name: "🔨 Build win32 and update the tap 🍺"
runs-on: fw13-ubuntu-latest
runs-on: msi-walmart
container: git.tesses.org/tesses50/global-containers-mingw:latest
steps:
- uses: actions/checkout@v4
url: https://git.tesses.org/tesses50/tessesframework.git
- run: |
mkdir artifacts
env -C Packaging/WinToolsIntel bash build.sh
@@ -58,31 +60,36 @@ jobs:
git push
build-packages:
name: "🔨 Build for everything else"
runs-on: fw13-ubuntu-latest
strategy:
fail-fast: true
matrix:
include:
- container: "git.tesses.org/tesses50/linux-x64:latest"
puter: "msi-walmart"
nfpmName: "amd64"
debianName: "x86_64-linux-gnu"
- container: "git.tesses.org/tesses50/linux-x86:latest"
puter: "asrock"
nfpmName: "386"
debianName: "i386-linux-gnu"
- container: "git.tesses.org/tesses50/linux-arm:latest"
puter: "fw13-ubuntu-latest"
nfpmName: "arm7"
debianName: "arm-linux-gnueabihf"
- container: "git.tesses.org/tesses50/linux-arm64:latest"
puter: "msi-walmart"
nfpmName: "arm64"
debianName: "aarch64-linux-gnu"
- container: "git.tesses.org/tesses50/linux-riscv64:latest"
nfpmName: "riscv64"
debianName: "riscv64-linux-gnu"
puter: "asrock"
runs-on: ${{ matrix.puter }}
container:
image: ${{ matrix.container }}
steps:
- uses: actions/checkout@v4
url: https://git.tesses.org/tesses50/tessesframework.git
- run: bash build-for-nfpm.sh ${{ matrix.nfpmName }} ${{ matrix.debianName }}
- uses: akkuman/gitea-release-action@v1
env:
-1
View File
@@ -7,7 +7,6 @@ set(CMAKE_CXX_STANDARD 17)
include(GNUInstallDirs)
include(CheckLibraryExists)
include(FetchContent)
include(cmake/sources.cmake)
include(cmake/options.cmake)
set(CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS ON)
+1
View File
@@ -22,6 +22,7 @@ Tesses Framework
- EventLoop
- Basic crytography using mbedtls
- a LCG Random number generator (do not use for crypto, use it for non security things like number guessing games and stuff, where rand would be ok)
- Simple HTML parser
## Supported Platforms
- Linux
+84
View File
@@ -0,0 +1,84 @@
#include "TessesFramework/TessesFramework.hpp"
using namespace Tesses::Framework::Streams;
using namespace Tesses::Framework::Serialization;
using namespace Tesses::Framework::TextStreams;
Json::JToken Html2JToken(std::shared_ptr<Html::HtmlNode> node) {
if (node->isText) {
return node->text_or_tag;
} else {
Json::JObject obj;
obj.SetValue("Tag", node->text_or_tag);
Json::JArray attrs;
for (auto &item : node->attributes) {
if (item.second)
attrs.Add(Json::JObject(
{{"Key", item.first}, {"Value", item.second.value()}}));
else
attrs.Add(
Json::JObject({{"Key", item.first}, {"Value", nullptr}}));
}
obj.SetValue("Attributes", attrs);
Json::JArray childern;
for (auto &item : node->children)
childern.Add(Html2JToken(item));
obj.SetValue("Children", childern);
return obj;
}
}
std::shared_ptr<FileStream> OpenWrite(std::string dest) {
if (dest == "-") {
return std::make_shared<FileStream>(stdout, false, "w");
} else {
auto strm = std::make_shared<FileStream>(dest, "w");
if (!strm->CanWrite()) {
return nullptr;
}
return strm;
}
}
std::shared_ptr<FileStream> OpenRead(std::string src) {
if (src == "-") {
return std::make_shared<FileStream>(stdin, false, "r");
} else {
auto strm = std::make_shared<FileStream>(src, "r");
if (!strm->CanRead()) {
return nullptr;
}
return strm;
}
}
int main(int argc, char **argv) {
if (argc < 3) {
std::cout << "USAGE: " << argv[0] << " SRC DEST" << std::endl;
std::cout << "SRC: html file or - for stdin to convert to json doc"
<< std::endl;
std::cout << "DEST: json file or - for stdout" << std::endl;
return 0;
}
auto src = OpenRead(argv[1]);
auto dest = OpenWrite(argv[2]);
if (src == nullptr) {
std::cerr << "ERROR: Input could not be read" << std::endl;
return 1;
}
if (dest == nullptr) {
std::cerr << "ERROR: Output could not be read" << std::endl;
return 1;
}
StreamWriter writer(dest);
auto html = Html::Parse(std::make_shared<StreamReader>(src));
writer.WriteLine(Json::Json::Encode(Html2JToken(html)));
return 0;
}
+77
View File
@@ -0,0 +1,77 @@
#include "TessesFramework/TessesFramework.hpp"
using namespace Tesses::Framework::Streams;
using namespace Tesses::Framework::Serialization;
using namespace Tesses::Framework::TextStreams;
std::shared_ptr<Stream> OpenRead(std::string src) {
if (src == "-") {
return std::make_shared<FileStream>(stdin, false, "r");
} else if (src.find("http://") == 0 || src.find("https://") == 0) {
Tesses::Framework::Http::HttpRequest req;
req.url = src;
req.method = "GET";
Tesses::Framework::Http::HttpResponse resp(req);
if ((int)resp.statusCode < 200 || (int)resp.statusCode > 299) {
throw std::runtime_error(
"Response does not indicate success: " +
std::to_string((int)resp.statusCode) + " " +
Tesses::Framework::Http::HttpUtils::StatusCodeString(
resp.statusCode) +
".");
}
return resp.ReadAsStream();
} else {
auto strm = std::make_shared<FileStream>(src, "r");
if (!strm->CanRead()) {
return nullptr;
}
return strm;
}
}
int main(int argc, char **argv) {
if (argc < 2) {
std::cout << "USAGE: " << argv[0] << " SRC" << std::endl;
std::cout << "SRC: html file or url or - for stdin" << std::endl;
return 0;
}
auto src = OpenRead(argv[1]);
if (src == nullptr) {
std::cerr << "ERROR: Input could not be read" << std::endl;
return 1;
}
auto html = Html::Parse(std::make_shared<StreamReader>(src));
std::vector<std::shared_ptr<const Html::HtmlNode>> titleTags;
html->FindNodes("title", titleTags);
for (auto &title : titleTags) {
Tesses::Framework::Console::WriteView("Title: ");
Tesses::Framework::Console::WriteLine(title->InnerText());
break;
}
std::vector<std::shared_ptr<const Html::HtmlNode>> metaTags;
html->FindNodes("meta", metaTags);
for (auto &meta : metaTags) {
std::optional<std::string> prop, content;
for (auto &[k, v] : meta->attributes) {
if (k == "property" && v)
prop = *v;
if (k == "content" && v)
content = *v;
}
if (prop && content && prop->find("og:") == 0) {
Tesses::Framework::Console::WriteView(prop.value());
Tesses::Framework::Console::WriteView(": ");
Tesses::Framework::Console::WriteLineView(content.value());
}
}
return 0;
}
+4 -3
View File
@@ -39,7 +39,7 @@ int main(int argc, char **argv) {
Args args(argc, argv);
if (args.positional.size() < 2)
usage(args);
int64_t num;
for (auto &opt : args.options) {
if (opt.first == "tracker")
trackers.push_back(opt.second);
@@ -49,8 +49,9 @@ int main(int argc, char **argv) {
comment = opt.second;
else if (opt.first == "created_by")
created_by = opt.second;
else if (opt.first == "piece_length")
pieceLength = std::stoll(opt.second);
else if (opt.first == "piece_length" &&
Serialization::BitConverter::TryParseSigned(opt.second, num))
pieceLength = num;
}
for (auto &flag : args.flags) {
+3
View File
@@ -1,5 +1,8 @@
# Changelog
## 0.0.8
Fixed a stoll bug that was there since day one, html parser, fix pkgconfig
## 0.0.7
Fix license compliance and fix things up, add reverse proxy, fix some security flaws with HttpUtils
+9
View File
@@ -73,4 +73,13 @@ add_executable(tsimplereverseproxy apps/tsimplereverseproxy.cpp)
target_link_libraries(tsimplereverseproxy PUBLIC tessesframework)
install(TARGETS tsimplereverseproxy DESTINATION "${CMAKE_INSTALL_BINDIR}")
add_executable(thtml2json apps/thtml2json.cpp)
target_link_libraries(thtml2json PUBLIC tessesframework)
install(TARGETS thtml2json DESTINATION "${CMAKE_INSTALL_BINDIR}")
add_executable(topengraph apps/topengraph.cpp)
target_link_libraries(topengraph PUBLIC tessesframework)
install(TARGETS topengraph DESTINATION "${CMAKE_INSTALL_BINDIR}")
endif()
+6 -8
View File
@@ -18,21 +18,19 @@ if(NOT mbedtls_POPULATED)
set(BUILD_SHARED_LIBS OFF CACHE BOOL "Build static" FORCE)
if(NOT TESSESFRAMEWORK_STATIC)
set(CMAKE_POSITION_INDEPENDENT_CODE ON CACHE BOOL "Build PIC" FORCE)
set(CMAKE_SKIP_INSTALL_RULES ON)
set(DISABLE_PACKAGE_CONFIG_AND_INSTALL ON CACHE BOOL "" FORCE)
endif()
FetchContent_Populate(mbedtls)
if(NOT TESSESFRAMEWORK_STATIC)
install(CODE "set(CMAKE_INSTALL_LOCAL_ONLY TRUE)" ALL_COMPONENTS)
endif()
add_subdirectory(${mbedtls_SOURCE_DIR} ${mbedtls_BINARY_DIR})
if(NOT TESSESFRAMEWORK_STATIC)
set(CMAKE_SKIP_INSTALL_RULES OFF)
add_subdirectory(${mbedtls_SOURCE_DIR} ${mbedtls_BINARY_DIR} EXCLUDE_FROM_ALL)
else()
add_subdirectory(${mbedtls_SOURCE_DIR} ${mbedtls_BINARY_DIR})
endif()
endif()
else()
if(TESSESFRAMEWORK_MBEDTLS_PORTLIB STREQUAL "")
+1
View File
@@ -22,6 +22,7 @@ src/Serialization/Json.cpp
src/Serialization/SQLite.cpp
src/Serialization/BitConverter.cpp
src/Serialization/Bencode.cpp
src/Serialization/Html.cpp
src/Platform/Environment.cpp
src/Platform/Process.cpp
src/Streams/FileStream.cpp
+1 -1
View File
@@ -1,3 +1,3 @@
set(TESSESFRAMEWORK_MAJOR_VERSION 0)
set(TESSESFRAMEWORK_MINOR_VERSION 0)
set(TESSESFRAMEWORK_PATCH_VERSION 7)
set(TESSESFRAMEWORK_PATCH_VERSION 8)
+1 -1
View File
@@ -98,7 +98,7 @@ class ServerContext {
void SendErrorPage(bool showPath);
void SendNotFound();
void SendBadRequest();
void SendException(std::exception &ex);
void SendException(const std::exception &ex);
void SendServerSentEvents(std::shared_ptr<ServerSentEvents> sse);
std::shared_ptr<Tesses::Framework::Streams::Stream> OpenResponseStream();
std::shared_ptr<Tesses::Framework::Streams::Stream> OpenRequestStream();
+4 -2
View File
@@ -157,9 +157,9 @@ class Uri {
std::string GetPathAndQuery();
uint16_t GetPort();
std::string HostPort();
bool Relative(std::string url, Uri &uri);
bool Relative(std::string_view url, Uri &uri);
std::string ToString();
static bool TryParse(std::string url, Uri &uri);
static bool TryParse(std::string_view url, Uri &uri);
std::string scheme;
std::string host;
uint16_t port;
@@ -199,6 +199,8 @@ class HttpUtils {
static std::string UrlPathEncode(std::string_view v,
bool ignoreSpace = false);
static std::string HtmlEncode(std::string_view v);
// not all escapes are supported
static std::string HtmlDecode(std::string_view v);
static std::string HtmlP(std::string_view text);
static void SplitString(std::vector<std::string> &out,
std::string_view text, std::string_view delimiter,
@@ -83,6 +83,12 @@ class BitConverter {
static void FromUuid(uint8_t &b, const Uuid &uuid);
static bool TryParseSigned(const std::string &str, int64_t &val,
int base = 10);
static bool TryParseUnsigned(const std::string &str, uint64_t &val,
int base = 10);
static bool TryParseDouble(const std::string &str, double &val);
static inline bool IsLittleEndian() {
uint8_t a[2];
a[0] = 0x01;
@@ -0,0 +1,71 @@
/*
TessesFramework a library to make C++ easier for me, used in CrossLang:
https://git.tesses.org/tesses50/crosslang
Copyright (C) 2026 Mike Nolan
SPDX-License-Identifier: GPL-3.0-or-later WITH TessesFramework-Exception-1.0
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
#pragma once
#include "../TextStreams/TextReader.hpp"
namespace Tesses::Framework::Serialization::Html {
struct HtmlNode;
struct HtmlNode : public std::enable_shared_from_this<HtmlNode> {
std::string text_or_tag;
std::vector<std::pair<std::string, std::optional<std::string>>> attributes;
std::vector<std::shared_ptr<HtmlNode>> children;
bool isText = false;
void FindNodes(std::string_view tag,
std::vector<std::shared_ptr<const HtmlNode>> &nodes) const;
std::vector<std::shared_ptr<const HtmlNode>>
FindNodes(std::string_view tag) const;
void ToString(std::string &str) const;
std::string ToString() const;
void InnerText(std::string &str) const;
std::string InnerText() const;
static std::shared_ptr<HtmlNode> FromText(std::string_view text);
static std::shared_ptr<HtmlNode>
FromNode(std::string_view name,
std::initializer_list<
std::pair<std::string, std::optional<std::string>>>
attrs,
std::initializer_list<std::shared_ptr<HtmlNode>> nodes);
static std::shared_ptr<HtmlNode> FromNode(
std::string_view name,
std::vector<std::pair<std::string, std::optional<std::string>>> attrs,
std::vector<std::shared_ptr<HtmlNode>> nodes);
template <typename IttrAttrs, typename IttrNodes>
static std::shared_ptr<HtmlNode>
FromNode(std::string_view name, IttrAttrs attrBegin, IttrAttrs attrEnd,
IttrNodes nodesBegin, IttrNodes nodesEnd) {
auto node = std::make_shared<HtmlNode>();
node->isText = false;
node->text_or_tag = std::string{name};
node->attributes.insert(node->attributes.end(), attrBegin, attrEnd);
node->children.insert(node->children.end(), nodesBegin, nodesEnd);
return node;
}
};
std::shared_ptr<HtmlNode>
Parse(std::shared_ptr<TextStreams::TextReader> reader);
} // namespace Tesses::Framework::Serialization::Html
@@ -50,6 +50,7 @@
#include "Random.hpp"
#include "Serialization/Bencode.hpp"
#include "Serialization/BitConverter.hpp"
#include "Serialization/Html.hpp"
#include "Serialization/Json.hpp"
#include "Serialization/SQLite.hpp"
#include "Streams/BufferedStream.hpp"
+2
View File
@@ -11,6 +11,8 @@ set(PKGCONFIG_DEPS "")
endif()
configure_file(tessesframework.pc.in tessesframework.pc @ONLY)
message(NOTICE "SrcFile ${CMAKE_CURRENT_BINARY_DIR}/tessesframework.pc")
message(NOTICE "Outdir ${CMAKE_INSTALL_LIBDIR}/pkgconfig")
install(FILES
${CMAKE_CURRENT_BINARY_DIR}/tessesframework.pc
DESTINATION ${CMAKE_INSTALL_LIBDIR}/pkgconfig)
+99 -39
View File
@@ -39,6 +39,8 @@
#include "TessesFramework/Filesystem/VFS.hpp"
#include "TessesFramework/Filesystem/VFSFix.hpp"
// clang-format on
#include "TessesFramework/Serialization/BitConverter.hpp"
using FileStream = Tesses::Framework::Streams::FileStream;
using Stream = Tesses::Framework::Streams::Stream;
using SeekOrigin = Tesses::Framework::Streams::SeekOrigin;
@@ -607,12 +609,34 @@ uint16_t HttpServer::GetPort() {
return server->GetPort();
return 0;
}
std::shared_ptr<Stream> ServerContext::OpenResponseStream() {
if (sent)
return nullptr;
int64_t length = -1;
if (!this->responseHeaders.TryGetFirstInt("Content-Length", length))
length = -1;
try {
if (!this->responseHeaders.TryGetFirstInt("Content-Length", length))
length = -1;
} catch (std::invalid_argument &invalid) {
std::string error =
"<!DOCTYPE html><html lang=\"en\"><head><meta "
"charset=\"UTF-8\"> <meta name=\"viewport\" "
"content=\"width=device-width, "
"initial-scale=1.0\"><title>Internal Server Error at " +
HttpUtils::HtmlEncode(this->originalPath) +
"</title><meta name=\"color-scheme\" content=\"dark "
"light\"></head><body><h1>Internal Server Error at " +
HttpUtils::HtmlEncode(this->originalPath) +
"</h1><p>what(): " + HttpUtils::HtmlEncode(invalid.what()) +
"</p></body></html>";
this->responseHeaders.SetValue("Content-Length",
static_cast<int64_t>(error.size()));
this->statusCode = StatusCode::InternalServerError;
this->WriteHeaders();
this->strm->WriteBlock(reinterpret_cast<const uint8_t *>(error.data()),
error.size());
return nullptr;
}
if (this->version == "HTTP/1.1" && length == -1)
this->responseHeaders.SetValue("Transfer-Encoding", "chunked");
@@ -648,6 +672,10 @@ void HttpServer::StartAccepting() {
uint16_t port;
auto sock = svr->GetStream(ip, port);
if (!TF_IsRunning()) // we need to die, that continue is not what we
// want if we must die from being done
break;
TF_LOG("New Host IP: " + ip + ":" + std::to_string(port));
if (sock == nullptr) {
@@ -708,9 +736,14 @@ ServerContext::ServerContext(std::shared_ptr<Stream> strm, bool debug)
}
std::shared_ptr<Stream> ServerContext::GetStream() { return this->strm; }
void ServerContext::SendBytes(std::vector<uint8_t> buff) {
std::shared_ptr<MemoryStream> strm = std::make_shared<MemoryStream>(false);
strm->GetBuffer() = buff;
SendStream(strm);
if (sent)
return;
this->responseHeaders.SetValue("Content-Length",
static_cast<int64_t>(buff.size()));
auto resp = OpenResponseStream();
if (resp)
resp->WriteBlock(buff.data(), buff.size());
}
ServerContext &ServerContext::WithLastModified(Date::DateTime dt) {
this->responseHeaders.SetValue("Last-Modified", dt);
@@ -718,11 +751,13 @@ ServerContext &ServerContext::WithLastModified(Date::DateTime dt) {
}
void ServerContext::SendText(std::string text) {
std::shared_ptr<MemoryStream> strm = std::make_shared<MemoryStream>(false);
auto &buff = strm->GetBuffer();
buff.insert(buff.end(), text.begin(), text.end());
SendStream(strm);
if (sent)
return;
this->responseHeaders.SetValue("Content-Length",
static_cast<int64_t>(text.size()));
auto resp = OpenResponseStream();
if (resp)
resp->WriteBlock(reinterpret_cast<uint8_t *>(text.data()), text.size());
}
void ServerContext::SendErrorPage(bool showPath) {
if (sent)
@@ -792,22 +827,34 @@ void ServerContext::SendStream(std::shared_ptr<Stream> strm) {
int64_t begin = 0;
int64_t end = -1;
if (dash.size() == 1 &&
res[0].find_first_of('-') != std::string::npos) {
// NUMBER-
begin = std::stoll(dash[0]);
} else if (dash.size() == 2) {
if (dash.size() == 2) {
// NUMBER-NUMBER
// or
//-NUMBER
if (dash[0].size() > 0) {
// NUMBER-NUMBER
begin = std::stoll(dash[0]);
end = std::stoll(dash[1]);
} else {
if (dash[0].empty()) {
//-NUMBER
end = std::stoll(dash[1]);
if (!Serialization::BitConverter::TryParseSigned(
dash[1], end))
throw std::runtime_error(
"Failed to parse the end of the Range header");
} else if (dash[1].empty()) {
if (!Serialization::BitConverter::TryParseSigned(
dash[0], begin))
throw std::runtime_error(
"Failed to parse the end of the Range header");
} else {
// NUMBER-NUMBER
if (!Serialization::BitConverter::TryParseSigned(
dash[0], begin))
throw std::runtime_error(
"Failed to parse the beginning of the Range "
"header");
if (!Serialization::BitConverter::TryParseSigned(
dash[1], end))
throw std::runtime_error(
"Failed to parse the end of the Range header");
}
} else {
this->statusCode = BadRequest;
@@ -886,6 +933,8 @@ void ServerContext::SendStream(std::shared_ptr<Stream> strm) {
} else {
auto chunkedStream = this->OpenResponseStream();
if (!chunkedStream)
throw std::runtime_error("Can't open response stream");
if (method != "HEAD")
strm->CopyTo(chunkedStream);
@@ -945,24 +994,28 @@ ServerContext &ServerContext::WithStatusCode(StatusCode code) {
this->statusCode = code;
return *this;
}
void ServerContext::SendException(std::exception &ex) {
if (this->debug) {
void ServerContext::SendException(const std::exception &ex) {
try {
if (this->debug) {
this->WithMimeType("text/html")
.WithStatusCode(StatusCode::InternalServerError)
.SendText("<!DOCTYPE html><html lang=\"en\"><head><meta "
"charset=\"UTF-8\"> <meta name=\"viewport\" "
"content=\"width=device-width, "
"initial-scale=1.0\"><title>Internal Server Error at " +
HttpUtils::HtmlEncode(this->originalPath) +
"</title><meta name=\"color-scheme\" content=\"dark "
"light\"></head><body><h1>Internal Server Error at " +
HttpUtils::HtmlEncode(this->originalPath) +
"</h1><p>what(): " + HttpUtils::HtmlEncode(ex.what()) +
"</p></body></html>");
} else {
this->WithStatusCode(StatusCode::InternalServerError)
.SendErrorPage(true);
this->WithMimeType("text/html")
.WithStatusCode(StatusCode::InternalServerError)
.SendText(
"<!DOCTYPE html><html lang=\"en\"><head><meta "
"charset=\"UTF-8\"> <meta name=\"viewport\" "
"content=\"width=device-width, "
"initial-scale=1.0\"><title>Internal Server Error at " +
HttpUtils::HtmlEncode(this->originalPath) +
"</title><meta name=\"color-scheme\" content=\"dark "
"light\"></head><body><h1>Internal Server Error at " +
HttpUtils::HtmlEncode(this->originalPath) +
"</h1><p>what(): " + HttpUtils::HtmlEncode(ex.what()) +
"</p></body></html>");
} else {
this->WithStatusCode(StatusCode::InternalServerError)
.SendErrorPage(true);
}
} catch (...) {
}
}
@@ -1106,6 +1159,13 @@ void HttpServer::Process(std::shared_ptr<Stream> strm,
ctx.requestHeaders.TryGetFirst("Content-Type", type) &&
type == "application/x-www-form-urlencoded" &&
ctx.requestHeaders.TryGetFirstInt("Content-Length", length)) {
if (length <= 0) {
ctx.statusCode = StatusCode::BadRequest;
ctx.SendErrorPage(true);
return;
}
size_t len = (size_t)length;
std::vector<uint8_t> buffer(len);
len = bStrm->ReadBlock(buffer.data(), len);
+152 -21
View File
@@ -21,14 +21,19 @@
#include "TessesFramework/Http/HttpUtils.hpp"
#include "TessesFramework/Filesystem/VFS.hpp"
#include "TessesFramework/Serialization/BitConverter.hpp"
#include "TessesFramework/Text/StringConverter.hpp"
#include <algorithm>
#include <iostream>
#include <sstream>
using VFSPath = Tesses::Framework::Filesystem::VFSPath;
namespace Tesses::Framework::Http {
bool Uri::Relative(std::string url, Uri &uri) {
auto index = url.find_first_of("//");
bool Uri::Relative(std::string_view url, Uri &uri) {
auto path = this->path;
if (path.empty())
path = "/";
auto index = url.find("//");
if (index != std::string::npos) {
if (Uri::TryParse(url, uri)) {
if (index == 0)
@@ -63,7 +68,10 @@ bool Uri::Relative(std::string url, Uri &uri) {
auto fourthPart = HttpUtils::SplitString(thirdPart[1], "?", 2);
VFSPath p = VFSPath(this->path, fourthPart[0]);
VFSPath p =
VFSPath(path, (path.back() == '/' || fourthPart[0].empty())
? fourthPart[0]
: "../" + fourthPart[0]);
uri.path =
p.CollapseRelativeParents().ToString(); // this should be safe
if (fourthPart.size() == 2) {
@@ -98,21 +106,33 @@ uint16_t Uri::GetPort() {
return 69;
return 0;
}
bool Uri::TryParse(std::string url, Uri &uri) {
bool Uri::TryParse(std::string_view url, Uri &uri) {
uri.scheme = "";
uri.port = 0;
auto firstPart = HttpUtils::SplitString(url, "//", 2);
if (firstPart.size() == 2)
uri.scheme = firstPart[0];
else if (firstPart.empty())
else
return false;
auto secondPart = HttpUtils::SplitString(
firstPart.size() == 2 ? firstPart[1] : firstPart[0], "/", 2);
auto secondPart = HttpUtils::SplitString(firstPart[1], "/", 2);
if (secondPart.size() == 1) {
uri.path = "/";
auto hp = secondPart[0];
auto fragPart = HttpUtils::SplitString(hp, "#", 2);
if (fragPart.size() == 2) {
uri.hash = fragPart[1];
hp = fragPart[0];
}
auto queryPart = HttpUtils::SplitString(hp, "?", 2);
if (queryPart.size() == 2) {
HttpUtils::QueryParamsDecode(uri.query, queryPart[1]);
hp = queryPart[0];
}
secondPart[0] = hp;
} else if (secondPart.size() == 2) {
auto thirdPart = HttpUtils::SplitString(secondPart[1], "#", 2);
if (thirdPart.empty())
@@ -135,14 +155,25 @@ bool Uri::TryParse(std::string url, Uri &uri) {
if (hostPortPart.empty())
return false;
if (hostPortPart.size() == 2) {
uri.port = (uint16_t)std::stoul(hostPortPart[1]);
uint64_t portNum;
if (!Serialization::BitConverter::TryParseUnsigned(hostPortPart[1],
portNum))
return false;
if (portNum > 65535)
return false;
uri.port = static_cast<uint16_t>(portNum);
}
uri.host = hostPortPart[0];
return true;
}
Uri::Uri() : query(true) {}
std::string Uri::GetPathAndQuery() { return this->path + this->GetQuery(); }
std::string Uri::GetPathAndQuery() {
if (this->path.empty() || this->path.front() != '/')
return '/' + this->path + this->GetQuery();
return this->path + this->GetQuery();
}
std::string Uri::GetQuery() {
if (this->query.kvp.empty())
return "";
@@ -532,9 +563,16 @@ std::string HttpUtils::HtmlP(std::string_view text) {
auto flush = [&]() -> void {
if (!builder.empty()) {
// Collect trailing punctuation
std::string trailing;
static const char *punct = ".,;:!?'\"";
while (!builder.empty() && std::strchr(punct, builder.back())) {
trailing.insert(trailing.begin(), builder.back());
builder.pop_back();
}
if (builder.find("http://") == 0 || builder.find("https://") == 0 ||
builder.find("ftp://") == 0 || builder.find("ftps://") == 0 ||
builder.find("magnet:") == 0 || builder.find("btmh:") == 0) {
builder.find("magnet:") == 0) {
newText += "<a href=\"" + HttpUtils::HtmlEncode(builder) +
"\">" + HttpUtils::HtmlEncode(builder) + "</a>";
} else if (builder.find("mailto:") == 0) {
@@ -548,6 +586,7 @@ std::string HttpUtils::HtmlP(std::string_view text) {
} else {
newText += HttpUtils::HtmlEncode(builder);
}
newText += HttpUtils::HtmlEncode(trailing);
builder = "";
}
};
@@ -564,7 +603,7 @@ std::string HttpUtils::HtmlP(std::string_view text) {
break;
case '\t':
flush();
newText += "&tab;";
newText += "&nbsp;&nbsp;&nbsp;&nbsp;";
break;
case '\r':
flush();
@@ -578,6 +617,106 @@ std::string HttpUtils::HtmlP(std::string_view text) {
return newText;
}
static std::unordered_map<std::string, std::string> htmlencodings = {
{"&quot;", "\""},
{"&apos;", "'"},
{"&amp;", "&"},
{"&lt;", "<"},
{"&gt;", ">"},
{"&nbsp;", "\xC2\xA0"},
{"&copy;", "\xC2\xA9"},
{"&reg;", "\xC2\xAE"},
{"&trade;", "\xE2\x84\xA2"},
{"&mdash;", "\xE2\x80\x94"},
{"&ndash;", "\xE2\x80\x93"},
{"&hellip;", "\xE2\x80\xA6"},
{"&rsquo;", "\xE2\x80\x99"},
{"&lsquo;", "\xE2\x80\x98"},
{"&rdquo;", "\xE2\x80\x9D"},
{"&ldquo;", "\xE2\x80\x9C"},
{"&euro;", "\xE2\x82\xAC"},
{"&pound;", "\xC2\xA3"},
{"&yen;", "\xC2\xA5"},
{"&deg;", "\xC2\xB0"},
{"&plusmn;", "\xC2\xB1"},
{"&times;", "\xC3\x97"},
{"&divide;", "\xC3\xB7"},
{"&laquo;", "\xC2\xAB"},
{"&raquo;", "\xC2\xBB"},
{"&bull;", "\xE2\x80\xA2"},
{"&middot;", "\xC2\xB7"},
{"&frac12;", "\xC2\xBD"},
{"&frac14;", "\xC2\xBC"},
{"&frac34;", "\xC2\xBE"},
};
std::string HttpUtils::HtmlDecode(std::string_view view) {
std::string myHtml = {};
std::string tmp = {};
bool inEscaped = false;
for (size_t i = 0; i < view.size(); i++) {
if (inEscaped) {
if (view[i] == ';') {
inEscaped = false;
if (tmp.size() > 2 && tmp[1] == '#') //&#N
{
if (tmp[2] == 'x' || tmp[2] == 'X') {
auto res = tmp.substr(3);
uint64_t code;
if (Serialization::BitConverter::TryParseUnsigned(
res, code, 16)) {
if (code <= 0x10FFFF &&
(code < 0xD800 || code > 0xDFFF)) {
Text::StringConverter::UTF8::FromUTF32(
myHtml, {static_cast<char32_t>(code)});
} else {
myHtml += "\xEF\xBF\xBD";
}
} else {
myHtml += "\xEF\xBF\xBD";
}
} else {
auto res = tmp.substr(2);
uint64_t code;
if (Serialization::BitConverter::TryParseUnsigned(
res, code, 10)) {
if (code <= 0x10FFFF &&
(code < 0xD800 || code > 0xDFFF)) {
Text::StringConverter::UTF8::FromUTF32(
myHtml, {static_cast<char32_t>(code)});
} else {
myHtml += "\xEF\xBF\xBD";
}
} else {
myHtml += "\xEF\xBF\xBD";
}
}
} else {
tmp += ';';
auto result = htmlencodings.find(tmp);
if (result != htmlencodings.end()) {
myHtml += result->second;
} else {
myHtml += "\xEF\xBF\xBD";
}
}
} else {
tmp += view[i];
}
} else {
if (view[i] == '&') {
tmp = "&";
inEscaped = true;
} else {
myHtml += view[i];
}
}
}
if (inEscaped)
myHtml += tmp;
return myHtml;
}
std::string HttpUtils::HtmlEncode(std::string_view html) {
std::string myHtml = {};
for (auto item : html) {
@@ -838,16 +977,8 @@ bool HttpDictionary::TryGetFirstInt(std::string key, int64_t &value) {
std::string val;
if (!TryGetFirst(key, val))
return false;
try {
size_t off = 0;
auto v = std::stoll(val, &off);
if (off != val.size())
return false;
value = v;
} catch (std::exception &ex) {
return false;
}
return true;
return Serialization::BitConverter::TryParseSigned(val, value);
}
bool HttpDictionary::TryGetFirstDate(std::string key, Date::DateTime &dt) {
std::string val;
+57
View File
@@ -284,4 +284,61 @@ void BitConverter::FromSint16LE(uint8_t &b, int16_t v) {
memcpy(&dest, &v, sizeof(uint16_t));
FromUint16LE(b, dest);
}
bool BitConverter::TryParseSigned(const std::string &str, int64_t &val,
int base) {
try {
if (str.find_first_not_of(" \t") != 0)
return false;
size_t pos = 0;
int64_t v = std::stoll(str, &pos, base);
if (pos < str.size())
return false;
val = v;
return true;
} catch (std::out_of_range &arg) {
return false;
} catch (std::invalid_argument &arg) {
return false;
}
}
bool BitConverter::TryParseUnsigned(const std::string &str, uint64_t &val,
int base) {
try {
if (str.find_first_not_of(" \t") != 0)
return false;
if (str[0] == '-' || str[0] == '+')
return false;
size_t pos = 0;
uint64_t v = std::stoull(str, &pos, base);
if (pos < str.size())
return false;
val = v;
return true;
} catch (std::out_of_range &arg) {
return false;
} catch (std::invalid_argument &arg) {
return false;
}
}
bool BitConverter::TryParseDouble(const std::string &str, double &val) {
try {
if (str.find_first_not_of(" \t") != 0)
return false;
size_t pos = 0;
double v = std::stod(str, &pos);
if (pos < str.size())
return false;
val = v;
return true;
} catch (std::out_of_range &arg) {
return false;
} catch (std::invalid_argument &arg) {
return false;
}
}
} // namespace Tesses::Framework::Serialization
+429
View File
@@ -0,0 +1,429 @@
#include "TessesFramework/Serialization/Html.hpp"
#include "TessesFramework/Console.hpp"
#include "TessesFramework/Http/HttpUtils.hpp"
#include <unordered_set>
namespace Tesses::Framework::Serialization::Html {
static std::unordered_set<std::string> html_nodes_with_no_body = {
"area", "base", "br", "col", "embed", "hr", "img",
"input", "link", "meta", "source", "track", "wbr",
};
std::string HtmlNode::ToString() const {
std::string text;
HtmlNode::ToString(text);
return text;
}
void HtmlNode::InnerText(std::string &str) const {
if (this->isText) {
str += this->text_or_tag;
} else {
for (const auto &item : this->children)
item->InnerText(str);
}
}
std::string HtmlNode::InnerText() const {
std::string innerText;
InnerText(innerText);
return innerText;
}
void HtmlNode::ToString(std::string &str) const {
if (this->isText) {
str += Http::HttpUtils::HtmlEncode(this->text_or_tag);
} else {
str += "<";
str += this->text_or_tag;
for (auto &attr : this->attributes) {
str += ' ';
str += attr.first;
if (attr.second) {
str += "=\"";
str += Http::HttpUtils::HtmlEncode(attr.second.value());
str += "\"";
}
}
str += ">";
if (html_nodes_with_no_body.count(this->text_or_tag) == 0) {
for (auto &child : this->children)
child->ToString(str);
str += "</";
str += this->text_or_tag;
str += ">";
}
}
}
std::shared_ptr<HtmlNode> HtmlNode::FromText(std::string_view text) {
auto node = std::make_shared<HtmlNode>();
node->isText = true;
node->text_or_tag = std::string{text};
return node;
}
std::shared_ptr<HtmlNode> HtmlNode::FromNode(
std::string_view name,
std::initializer_list<std::pair<std::string, std::optional<std::string>>>
attrs,
std::initializer_list<std::shared_ptr<HtmlNode>> nodes) {
return HtmlNode::FromNode(name, attrs.begin(), attrs.end(), nodes.begin(),
nodes.end());
}
std::shared_ptr<HtmlNode> HtmlNode::FromNode(
std::string_view name,
std::vector<std::pair<std::string, std::optional<std::string>>> attrs,
std::vector<std::shared_ptr<HtmlNode>> nodes) {
return HtmlNode::FromNode(name, attrs.begin(), attrs.end(), nodes.begin(),
nodes.end());
}
void HtmlNode::FindNodes(
std::string_view tag,
std::vector<std::shared_ptr<const HtmlNode>> &nodes) const {
if (!this->isText) {
if (this->text_or_tag == tag)
nodes.push_back(this->shared_from_this());
for (const auto &node : this->children)
node->FindNodes(tag, nodes);
}
}
std::vector<std::shared_ptr<const HtmlNode>>
HtmlNode::FindNodes(std::string_view tag) const {
std::vector<std::shared_ptr<const HtmlNode>> nodes;
FindNodes(tag, nodes);
return nodes;
}
enum class HtmlLexTokenType { SYMBOL, TEXT, STRING };
struct HtmlLexToken {
std::string text;
HtmlLexTokenType type;
HtmlLexToken(HtmlLexTokenType type, std::string_view text)
: text(text), type(type) {}
};
struct HtmlLexer {
std::shared_ptr<TextStreams::TextReader> reader;
std::vector<HtmlLexToken> &tokens;
int32_t _peeked = -1;
std::string builder;
bool ignoreSpaceMode = false;
std::string nodeName = {};
HtmlLexer(std::shared_ptr<TextStreams::TextReader> reader,
std::vector<HtmlLexToken> &tokens)
: reader(std::move(reader)), tokens(tokens) {}
int32_t Peek() {
if (_peeked != -1)
return _peeked;
_peeked = Read();
return _peeked;
}
int64_t Read() {
if (_peeked != -1) {
auto tmp = _peeked;
_peeked = -1;
return tmp;
}
return reader->ReadChar();
}
void Flush() {
if (!builder.empty()) {
this->tokens.emplace_back(HtmlLexTokenType::TEXT, builder);
if (ignoreSpaceMode && nodeName.empty())
nodeName = builder;
builder = "";
}
}
void SkipTo(std::string_view fmt) {
if (fmt.empty())
return;
std::string window;
window.resize(fmt.size());
while (fmt != window) {
int32_t read = Read();
if (read == -1)
break;
for (size_t i = 0; i < window.size() - 1; i++) {
window[i] = window[i + 1];
}
window.back() = static_cast<char>(read);
}
}
void Lex() {
int read;
while ((read = Read()) != -1) {
int peek = Peek();
switch (read) {
case '"':
case '\'': {
if (ignoreSpaceMode) {
Flush();
char quote = static_cast<char>(read);
std::string val;
int c;
while ((c = Read()) != -1 && c != quote)
val += static_cast<char>(c);
tokens.emplace_back(HtmlLexTokenType::STRING, val);
} else {
builder += static_cast<char>(read);
}
break;
}
case '/': {
if (ignoreSpaceMode && peek == '>') {
Flush();
Read();
ignoreSpaceMode = false;
tokens.emplace_back(HtmlLexTokenType::SYMBOL, "/>");
} else if (ignoreSpaceMode) {
builder += '/';
} else {
builder += '/';
}
break;
}
case '>': {
if (ignoreSpaceMode) {
Flush();
ignoreSpaceMode = false;
tokens.emplace_back(HtmlLexTokenType::SYMBOL, ">");
if (nodeName == "script" || nodeName == "style") {
nodeName = "_";
std::string builder2 = "";
bool eof = false;
while ((read = Read()) != -1) {
if (read == '<') {
peek = Peek();
if (peek == '/') {
Read();
break;
} else if (peek == -1) {
eof = true;
break;
} else {
builder2 += '<';
}
} else {
builder2 += static_cast<char>(read);
}
}
if (read != -1 && !eof) {
tokens.emplace_back(HtmlLexTokenType::TEXT,
builder2);
ignoreSpaceMode = true;
tokens.emplace_back(HtmlLexTokenType::SYMBOL, "</");
} else {
return;
}
}
} else {
builder += '>';
}
} break;
case '<': {
Flush();
if (peek == '!') {
//<!-- -->
//<!DOCTYPE html>
Read();
peek = Peek();
if (peek == '-') {
Read();
peek = Peek();
if (peek == '-') {
Read();
SkipTo("-->");
} else {
SkipTo(">");
}
} else {
SkipTo(">");
}
} else if (peek == '/') {
Read();
ignoreSpaceMode = true;
tokens.emplace_back(HtmlLexTokenType::SYMBOL, "</");
} else {
ignoreSpaceMode = true;
tokens.emplace_back(HtmlLexTokenType::SYMBOL, "<");
nodeName = {};
}
} break;
case '=': {
if (!ignoreSpaceMode) {
builder += "=";
} else {
Flush();
tokens.emplace_back(HtmlLexTokenType::SYMBOL, "=");
}
} break;
case ' ':
case '\n':
case '\t':
case '\r':
if (!ignoreSpaceMode) {
builder += static_cast<char>(read);
} else {
Flush();
}
break;
default:
builder += static_cast<char>(read);
break;
}
}
Flush();
}
};
struct HtmlParser {
std::vector<HtmlLexToken> &tokens;
size_t index = 0;
HtmlParser(std::vector<HtmlLexToken> &tokens) : tokens(tokens) {}
bool IsSymbol(std::string_view type) {
if (Done())
return false;
auto &obj = Peak();
if (obj.type != HtmlLexTokenType::SYMBOL)
return false;
if (obj.text != type)
return false;
index++;
return true;
}
bool Done() { return index >= tokens.size(); }
HtmlLexToken &Read() { return tokens[index++]; }
HtmlLexToken &Peak() { return tokens[index]; }
void Parse(std::string_view name, std::shared_ptr<HtmlNode> &node,
bool root = false) {
while (!Done()) {
auto &curToken = Peak();
switch (curToken.type) {
case HtmlLexTokenType::TEXT: {
Read();
node->children.push_back(HtmlNode::FromText(
Http::HttpUtils::HtmlDecode(curToken.text)));
} break;
case HtmlLexTokenType::SYMBOL: {
if (curToken.text == "<") {
Read();
if (Done()) {
return;
}
auto &val = Peak();
if (val.type == HtmlLexTokenType::TEXT) {
Read();
std::shared_ptr<HtmlNode> newNode =
std::make_shared<HtmlNode>();
newNode->isText = false;
newNode->text_or_tag = val.text;
while (!Done()) {
auto &k = Peak();
if (k.type == HtmlLexTokenType::TEXT) {
Read();
if (IsSymbol("=")) {
if (!Done()) {
auto &v = Peak();
if (v.type ==
HtmlLexTokenType::STRING) {
Read();
newNode->attributes.emplace_back(
k.text,
Http::HttpUtils::HtmlDecode(
v.text));
continue;
}
}
}
newNode->attributes.emplace_back(k.text,
std::nullopt);
continue;
}
if (k.type == HtmlLexTokenType::SYMBOL &&
(k.text == ">" || k.text == "/>"))
break;
}
if (!IsSymbol("/>")) {
IsSymbol(">");
if (html_nodes_with_no_body.count(val.text) == 0)
Parse(val.text, newNode);
}
node->children.push_back(newNode);
}
} else if (curToken.text == "</") {
Read();
if (Done()) {
node->children.push_back(HtmlNode::FromText("</"));
return;
}
auto &val = Peak();
Read();
IsSymbol(">");
if (val.text == name && !root)
return;
}
} break;
}
}
}
std::shared_ptr<HtmlNode> Parse() {
std::shared_ptr<HtmlNode> node = std::make_shared<HtmlNode>();
node->isText = false;
node->text_or_tag = ":root";
Parse(":root", node, true);
return node;
}
};
std::shared_ptr<HtmlNode>
Parse(std::shared_ptr<TextStreams::TextReader> reader) {
std::vector<HtmlLexToken> tokens;
HtmlLexer lexer(reader, tokens);
lexer.Lex();
HtmlParser parser(tokens);
return parser.Parse();
}
} // namespace Tesses::Framework::Serialization::Html