New FileSpec class to handle HTTP headers and query in filename (#4625

)

* Proof-of-concept file spec extension.

* explicit fs::path to string conversion

* initial filespec additions

* Move parsing to FileSpec.

* filepec conversion stuff - not working

* fix for fromString()

* pipelinemanager fixes. pipeline reader needs work

* ostream fix

* Add missed file.

* update to align with andrew's filespec

* assign m_filename earlier

* testing filespec creation

* fromString and pipelinereader updates

* cleanup filespec additions

* PR fixes 1

* PR fixes 2 - remove string constructor

* add FileSpec ctor to Connector

* remove filename-only ostream output

* add test and docs

* don't export filespec

---------

Co-authored-by: Andrew Bell <[email protected]>

doc/stages/filespec.md

@@ -0,0 +1,20 @@
+
+In addition to a string filename, HTTP headers and query parameters to be forwarded to remote endpoints
+can be specified within the filename option. As shown below, a JSON object can be substituted, with the
+'headers' and 'query' fields as JSON objects of key/value string pairs.
+```json
+{
+    "filename":
+    {
+        "path":"path to remote file [required]",
+        "headers":
+        {
+            "some_header_key": "HTTP header value"
+        },
+        "query":
+        {
+            "some_query_key": "HTTP query value"
+        }
+    }
+}
+```

io/private/connector/Connector.cpp

@@ -95,6 +95,18 @@ Connector::Connector(const std::string& filename, const StringMap& headers,
     }
 }
 
+Connector::Connector(const FileSpec& spec) :
+    m_arbiter(new arbiter::Arbiter),
+    m_headers(spec.m_headers),
+    m_query(spec.m_query),
+    m_filename(spec.m_path.string())
+{
+    if (m_headers.find("User-Agent") == m_headers.end())
+    {
+        m_headers.insert( std::make_pair("User-Agent", getUserAgent()));
+    }
+}
+
 
 std::string Connector::get(const std::string& path) const
 {

io/private/connector/Connector.hpp

@@ -35,6 +35,7 @@
 #pragma once
 
 #include <arbiter/arbiter.hpp>
+#include <pdal/FileSpec.hpp>
 
 using StringMap = std::map<std::string, std::string>;
 
@@ -55,6 +56,7 @@ class Connector
     Connector();
     Connector(const std::string& filename, const StringMap& headers, const StringMap& query);
     Connector(const StringMap& headers, const StringMap& query);
+    Connector(const FileSpec& spec);
 
     std::string get(const std::string& path) const;
     NL::json getJson(const std::string& path) const;

pdal/FileSpec.cpp

@@ -0,0 +1,167 @@
+/******************************************************************************
+* Copyright (c) 2025, Hobu Inc.
+*
+* All rights reserved.
+*
+* Redistribution and use in source and binary forms, with or without
+* modification, are permitted provided that the following
+* conditions are met:
+*
+*     * Redistributions of source code must retain the above copyright
+*       notice, this list of conditions and the following disclaimer.
+*     * Redistributions in binary form must reproduce the above copyright
+*       notice, this list of conditions and the following disclaimer in
+*       the documentation and/or other materials provided
+*       with the distribution.
+*     * Neither the name of Hobu, Inc. or Flaxen Geo Consulting nor the
+*       names of its contributors may be used to endorse or promote
+*       products derived from this software without specific prior
+*       written permission.
+*
+* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+* FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+* COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
+* OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+* AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY
+* OF SUCH DAMAGE.
+****************************************************************************/
+
+#include "FileSpec.hpp"
+
+#include <nlohmann/json.hpp>
+
+namespace pdal
+{
+
+namespace
+{
+
+bool extractStringMap(NL::json& node, StringMap& map)
+{
+    if (!node.is_object())
+        return false;
+    for (auto& [key, val] : node.items())
+    {
+        if (val.is_string())
+            map.insert({key, val});
+        else
+            return false;
+    }
+    return true;
+}
+
+} // unnamed namespace
+
+
+Utils::StatusWithReason FileSpec::ingest(const std::string& pathOrJson)
+{
+    NL::json json;
+    size_t pos = Utils::extractSpaces(pathOrJson, 0);
+    if (pathOrJson[pos] == '{' || pathOrJson[pos] == '[')
+    {
+        auto status = Utils::parseJson(pathOrJson, json);
+        if (!status)
+            return status;
+    }
+    // assuming input is a filename
+    else
+        json = NL::json(pathOrJson);
+
+    return parse(json);
+}
+
+Utils::StatusWithReason FileSpec::parse(NL::json& node)
+{
+    if (node.is_null())
+        return { -1, "'filename' argument contains no data" };
+    if (node.is_string())
+        m_path = node.get<std::string>();
+    else if (node.is_object())
+    {
+        auto status = extractPath(node);
+        if (!status)
+            return { -1, status.what() };
+        status = extractHeaders(node);
+        if (!status)
+            return { -1, status.what() };
+        status = extractQuery(node);
+        if (!status)
+            return { -1, status.what() };
+        if (!node.empty())
+            return { -1, "Invalid item in filename object: " + node.dump() };
+    }
+    else
+        return { -1, "'filename' must be specified as a string." };
+    return true;
+}
+
+Utils::StatusWithReason FileSpec::extractPath(NL::json& node)
+{
+    auto it = node.find("path");
+    if (it == node.end())
+        return { -1, "'filename' object must contain 'path' member." };
+    NL::json& val = *it;
+    if (!val.is_null())
+    {
+        if (val.is_string())
+            m_path = val.get<std::string>();
+        else
+            return { -1, "'filename' object 'path' member must be specified as a string." };
+        node.erase(it);
+    }
+    return true;
+}
+
+Utils::StatusWithReason FileSpec::extractQuery(NL::json& node)
+{
+    auto it = node.find("query");
+    if (it == node.end())
+        return true;
+    NL::json& val = *it;
+    if (!val.is_null())
+    {
+        if (!extractStringMap(val, m_query))
+            return { -1, "'filename' sub-argument 'query' must be an object of "
+                "string key-value pairs." };
+    }
+    node.erase(it);
+    return true;
+}
+
+Utils::StatusWithReason FileSpec::extractHeaders(NL::json& node)
+{
+    auto it = node.find("headers");
+    if (it == node.end())
+        return true;
+    NL::json& val = *it;
+    if (!val.is_null())
+    {
+        if (!extractStringMap(val, m_headers))
+            return { -1, "'filename' sub-argument 'headers' must be an object of "
+                "string key-value pairs." };
+    }
+    node.erase(it);
+    return true;
+}
+
+std::ostream& operator << (std::ostream& out, const FileSpec& spec)
+{
+    NL::json json;
+    json["path"] = spec.m_path.string();
+    if (!spec.m_headers.empty())
+        json["headers"] = spec.m_headers;
+    if (!spec.m_query.empty())
+        json["query"] = spec.m_query;
+
+    out << json;
+    return out;
+}
+
+
+} // namespace pdal

pdal/FileSpec.hpp

@@ -0,0 +1,85 @@
+/******************************************************************************
+* Copyright (c) 2025, Hobu Inc.
+*
+* All rights reserved.
+*
+* Redistribution and use in source and binary forms, with or without
+* modification, are permitted provided that the following
+* conditions are met:
+*
+*     * Redistributions of source code must retain the above copyright
+*       notice, this list of conditions and the following disclaimer.
+*     * Redistributions in binary form must reproduce the above copyright
+*       notice, this list of conditions and the following disclaimer in
+*       the documentation and/or other materials provided
+*       with the distribution.
+*     * Neither the name of Hobu, Inc. nor the names of its contributors
+*       may be used to endorse or promote products derived from this
+*       software without specific prior written permission.
+*
+* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+* FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+* COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
+* OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+* AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY
+* OF SUCH DAMAGE.
+****************************************************************************/
+
+#pragma once
+
+#include <filesystem>
+
+#include <pdal/PDALUtils.hpp>
+#include <pdal/pdal_types.hpp>
+
+#include <pdal/JsonFwd.hpp>
+
+using StringMap = std::map<std::string, std::string>;
+
+namespace pdal
+{
+
+class FileSpec
+{
+public:
+    FileSpec()
+    {}
+
+    bool valid() const
+    { return !m_path.empty(); }
+    bool onlyFilename() const
+    { return m_headers.empty() && m_query.empty(); }
+    Utils::StatusWithReason parse(NL::json& json);
+    // parse a user input string that could be a json spec or filename
+    Utils::StatusWithReason ingest(const std::string& pathOrJson);
+
+    friend std::ostream& operator << (std::ostream& out, const FileSpec& spec);
+
+private:
+    Utils::StatusWithReason extractPath(NL::json& node);
+    Utils::StatusWithReason extractQuery(NL::json& node);
+    Utils::StatusWithReason extractHeaders(NL::json& node);
+
+public:
+    std::filesystem::path m_path;
+    StringMap m_headers;
+    StringMap m_query;
+};
+
+namespace Utils
+{
+    template<>
+    inline StatusWithReason fromString(const std::string& s, FileSpec& spec)
+    {
+        return spec.ingest(s);
+    }
+}
+
+} // namespace pdal
+

pdal/Filter.hpp

@@ -67,6 +67,8 @@ class PDAL_EXPORT Filter : public virtual Stage
     virtual PointViewSet run(PointViewPtr view);
     virtual void filter(PointView& /*view*/)
     {}
+    virtual void assignParsedOptions() final
+    {}
 
     friend PDAL_EXPORT std::istream& operator>>(std::istream& in, Filter::WhereMergeMode& mode);
     friend PDAL_EXPORT std::ostream& operator<<(std::ostream& out, const Filter::WhereMergeMode& mode);

pdal/PDALUtils.cpp

@@ -516,5 +516,24 @@ double computeChamfer(PointViewPtr srcView, PointViewPtr candView)
     return sum1 + sum2;
 }
 
+StatusWithReason parseJson(const std::string& s, NL::json& json)
+{
+    try
+    {
+        json = NL::json::parse(s);
+    }
+    catch (NL::json::parse_error& err)
+    {
+        // Look for a right bracket -- this indicates the start of the
+        // actual message from the parse error.
+        std::string s(err.what());
+        auto pos = s.find(']');
+        if (pos != std::string::npos)
+            s = s.substr(pos + 1);
+        return { -1, s };
+    }
+    return true;
+}
+
 } // namespace Utils
 } // namespace pdal

pdal/PDALUtils.hpp

@@ -36,6 +36,7 @@
 
 #include <pdal/Metadata.hpp>
 #include <pdal/Dimension.hpp>
+#include <pdal/JsonFwd.hpp>
 #include <pdal/pdal_export.hpp>
 #include <pdal/util/Bounds.hpp>
 #include <pdal/util/Inserter.hpp>
@@ -280,7 +281,7 @@ double PDAL_EXPORT computeHausdorff(PointViewPtr srcView, PointViewPtr candView)
 std::pair<double, double> PDAL_EXPORT computeHausdorffPair(PointViewPtr srcView, PointViewPtr candView);
 double PDAL_EXPORT computeChamfer(PointViewPtr srcView, PointViewPtr candView);
 std::string PDAL_EXPORT tempFilename(const std::string& path);
-
+StatusWithReason PDAL_EXPORT parseJson(const std::string& s, NL::json& json);
 
 } // namespace Utils
 } // namespace pdal