Feature Multipart. WIP. Working multipart stream parser.

This commit is contained in:
lganzzzo 2019-07-13 12:00:00 +03:00
parent 427612adfd
commit 875f0f1378
10 changed files with 602 additions and 55 deletions

View File

@ -80,6 +80,8 @@ add_library(oatpp
oatpp/core/data/mapping/type/Type.hpp
oatpp/core/data/share/MemoryLabel.cpp
oatpp/core/data/share/MemoryLabel.hpp
oatpp/core/data/stream/BufferInputStream.cpp
oatpp/core/data/stream/BufferInputStream.hpp
oatpp/core/data/stream/ChunkedBuffer.cpp
oatpp/core/data/stream/ChunkedBuffer.hpp
oatpp/core/data/stream/Delegate.cpp
@ -141,6 +143,8 @@ add_library(oatpp
oatpp/web/client/HttpRequestExecutor.hpp
oatpp/web/client/RequestExecutor.cpp
oatpp/web/client/RequestExecutor.hpp
oatpp/web/mime/multipart/Part.cpp
oatpp/web/mime/multipart/Part.hpp
oatpp/web/mime/multipart/StatefulParser.cpp
oatpp/web/mime/multipart/StatefulParser.hpp
oatpp/web/protocol/CommunicationError.cpp

View File

@ -0,0 +1,75 @@
/***************************************************************************
*
* Project _____ __ ____ _ _
* ( _ ) /__\ (_ _)_| |_ _| |_
* )(_)( /(__)\ )( (_ _)(_ _)
* (_____)(__)(__)(__) |_| |_|
*
*
* Copyright 2018-present, Leonid Stryzhevskyi <lganzzzo@gmail.com>
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
***************************************************************************/
#include "BufferInputStream.hpp"
namespace oatpp { namespace data{ namespace stream {
BufferInputStream::BufferInputStream(const std::shared_ptr<base::StrBuffer>& memoryHandle, p_char8 data, v_io_size size)
: m_memoryHandle(memoryHandle)
, m_data(data)
, m_size(size)
, m_position(0)
, m_ioMode(IOMode::NON_BLOCKING)
{}
data::v_io_size BufferInputStream::read(void *data, data::v_io_size count) {
data::v_io_size desiredAmount = count;
if(desiredAmount > m_size - m_position) {
desiredAmount = m_size - m_position;
}
std::memcpy(data, &m_data[m_position], desiredAmount);
m_position += desiredAmount;
return desiredAmount;
}
void BufferInputStream::setInputStreamIOMode(IOMode ioMode) {
m_ioMode = ioMode;
}
IOMode BufferInputStream::getInputStreamIOMode() {
return m_ioMode;
}
std::shared_ptr<base::StrBuffer> BufferInputStream::getDataMemoryHandle() {
return m_memoryHandle;
}
p_char8 BufferInputStream::getData() {
return m_data;
}
v_io_size BufferInputStream::getDataSize() {
return m_size;
}
v_io_size BufferInputStream::getCurrentPosition() {
return m_position;
}
void BufferInputStream::resetPosition() {
m_position = 0;
}
}}}

View File

@ -0,0 +1,117 @@
/***************************************************************************
*
* Project _____ __ ____ _ _
* ( _ ) /__\ (_ _)_| |_ _| |_
* )(_)( /(__)\ )( (_ _)(_ _)
* (_____)(__)(__)(__) |_| |_|
*
*
* Copyright 2018-present, Leonid Stryzhevskyi <lganzzzo@gmail.com>
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
***************************************************************************/
#ifndef oatpp_data_stream_BufferInputStream_hpp
#define oatpp_data_stream_BufferInputStream_hpp
#include "Stream.hpp"
namespace oatpp { namespace data{ namespace stream {
class BufferInputStream : public InputStream {
private:
std::shared_ptr<base::StrBuffer> m_memoryHandle;
p_char8 m_data;
v_io_size m_size;
v_io_size m_position;
IOMode m_ioMode;
public:
/**
* Constructor.
* @param memoryHandle - buffer memory handle. May be nullptr.
* @param data - pointer to buffer data.
* @param size - size of the buffer.
*/
BufferInputStream(const std::shared_ptr<base::StrBuffer>& memoryHandle, p_char8 data, v_io_size size);
/**
* Read data from stream. <br>
* It is a legal case if return result < count. Caller should handle this!
* *Calls to this method are always NON-BLOCKING*
* @param data - buffer to read data to.
* @param count - size of the buffer.
* @return - actual number of bytes read. 0 - designates end of the buffer.
*/
data::v_io_size read(void *data, data::v_io_size count) override;
/**
* Not expected to be called because read method should always return correct amount or zero.
* @throws - `std::runtime_error`.
*/
oatpp::async::Action suggestInputStreamAction(data::v_io_size ioResult) override {
const char* message =
"Error. oatpp::data::stream::BufferInputStream::suggestOutputStreamAction() method is called.\n"
"No suggestions for BufferInputStream async I/O operations are needed.\n "
"BufferInputStream always satisfies call to read() method or returns 0 - as EOF.";
throw std::runtime_error(message);
}
/**
* Set stream I/O mode.
* @throws
*/
void setInputStreamIOMode(IOMode ioMode) override;
/**
* Get stream I/O mode.
* @return
*/
IOMode getInputStreamIOMode() override;
/**
* Get data memory handle.
* @return - data memory handle.
*/
std::shared_ptr<base::StrBuffer> getDataMemoryHandle();
/**
* Get pointer to data.
* @return - pointer to data.
*/
p_char8 getData();
/**
* Get data size.
* @return - data size.
*/
v_io_size getDataSize();
/**
* Get current data read position.
* @return - current data read position.
*/
v_io_size getCurrentPosition();
/**
* Reset current data read position to zero.
*/
void resetPosition();
};
}}}
#endif // oatpp_data_stream_BufferInputStream_hpp

View File

@ -40,6 +40,7 @@ ChunkedBuffer::ChunkedBuffer()
, m_chunkPos(0)
, m_firstEntry(nullptr)
, m_lastEntry(nullptr)
, m_ioMode(IOMode::NON_BLOCKING)
{}
ChunkedBuffer::~ChunkedBuffer() {

View File

@ -132,7 +132,7 @@ public:
/**
* Read data from stream up to count bytes, and return number of bytes actually read. <br>
* It is a legal case if return result < count. Caller should handle this!
* @param data - buffer to read dat to.
* @param data - buffer to read data to.
* @param count - size of the buffer.
* @return - actual number of bytes read.
*/

View File

@ -0,0 +1,140 @@
/***************************************************************************
*
* Project _____ __ ____ _ _
* ( _ ) /__\ (_ _)_| |_ _| |_
* )(_)( /(__)\ )( (_ _)(_ _)
* (_____)(__)(__)(__) |_| |_|
*
*
* Copyright 2018-present, Leonid Stryzhevskyi <lganzzzo@gmail.com>
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
***************************************************************************/
#include "Part.hpp"
#include "oatpp/core/parser/Caret.hpp"
#include <cstring>
namespace oatpp { namespace web { namespace mime { namespace multipart {
oatpp::String Part::parseContentDispositionValue(const char* key, p_char8 data, v_int32 size) {
parser::Caret caret(data, size);
if(caret.findText(key)) {
caret.inc(std::strlen(key));
parser::Caret::Label label(nullptr);
if(caret.isAtChar('"')) {
label = caret.parseStringEnclosed('"', '"', '\\');
} else if(caret.isAtChar('\'')) {
label = caret.parseStringEnclosed('\'', '\'', '\\');
} else {
label = caret.putLabel();
caret.findCharFromSet(" \t\n\r\f");
label.end();
}
if(label) {
return label.toString();
} else {
throw std::runtime_error("[oatpp::web::mime::multipart::Part::parseContentDispositionValue()]: Error. Can't parse value.");
}
}
return nullptr;
}
Part::Part(const Headers &headers,
const std::shared_ptr<data::stream::InputStream> &inputStream,
const oatpp::String inMemoryData,
data::v_io_size knownSize)
: m_headers(headers)
, m_inputStream(inputStream)
, m_inMemoryData(inMemoryData)
, m_knownSize(knownSize)
{
auto it = m_headers.find("Content-Disposition");
if(it != m_headers.end()) {
m_name = parseContentDispositionValue("name=", it->second.getData(), it->second.getSize());
if(!m_name) {
throw std::runtime_error("[oatpp::web::mime::multipart::Part::Part()]: Error. Part name is missing in 'Content-Disposition' header.");
}
m_filename = parseContentDispositionValue("filename=", it->second.getData(), it->second.getSize());
} else {
throw std::runtime_error("[oatpp::web::mime::multipart::Part::Part()]: Error. Missing 'Content-Disposition' header.");
}
}
Part::Part(const Headers& headers) : Part(headers, nullptr, nullptr, -1) {}
void Part::setDataInfo(const std::shared_ptr<data::stream::InputStream>& inputStream,
const oatpp::String inMemoryData,
data::v_io_size knownSize)
{
m_inputStream = inputStream;
m_inMemoryData = inMemoryData;
m_knownSize = knownSize;
}
oatpp::String Part::getName() const {
return m_name;
}
oatpp::String Part::getFilename() const {
return m_filename;
}
const Part::Headers& Part::getHeaders() const {
return m_headers;
}
oatpp::String Part::getHeader(const oatpp::data::share::StringKeyLabelCI_FAST &headerName) const {
auto it = m_headers.find(headerName);
if(it != m_headers.end()) {
return it->second.toString();
}
return nullptr;
}
std::shared_ptr<data::stream::InputStream> Part::getInputStream() const {
return m_inputStream;
}
oatpp::String Part::getInMemoryData() const {
return m_inMemoryData;
}
data::v_io_size Part::getKnownSize() const {
return m_knownSize;
}
}}}}

View File

@ -0,0 +1,136 @@
/***************************************************************************
*
* Project _____ __ ____ _ _
* ( _ ) /__\ (_ _)_| |_ _| |_
* )(_)( /(__)\ )( (_ _)(_ _)
* (_____)(__)(__)(__) |_| |_|
*
*
* Copyright 2018-present, Leonid Stryzhevskyi <lganzzzo@gmail.com>
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
***************************************************************************/
#ifndef oatpp_web_mime_multipart_Part_hpp
#define oatpp_web_mime_multipart_Part_hpp
#include "oatpp/core/data/share/MemoryLabel.hpp"
#include "oatpp/core/data/stream/Stream.hpp"
#include <unordered_map>
namespace oatpp { namespace web { namespace mime { namespace multipart {
/**
* One part of the multipart.
*/
class Part {
public:
/**
* Typedef for headers map. Headers map key is case-insensitive.
* `std::unordered_map` of &id:oatpp::data::share::StringKeyLabelCI_FAST; and &id:oatpp::data::share::StringKeyLabel;.
*/
typedef std::unordered_map<oatpp::data::share::StringKeyLabelCI_FAST, oatpp::data::share::StringKeyLabel> Headers;
private:
/**
* Parse value from the `Content-Disposition` header.
*/
static oatpp::String parseContentDispositionValue(const char* key, p_char8 data, v_int32 size);
private:
oatpp::String m_name;
oatpp::String m_filename;
Headers m_headers;
std::shared_ptr<data::stream::InputStream> m_inputStream;
oatpp::String m_inMemoryData;
data::v_io_size m_knownSize;
public:
/**
* Constructor.
* @param headers - headers of the part.
* @param inputStream - input stream of the part data.
* @param inMemoryData - possible in-memory data of the part. Same data as the referred by input stream. For convenience purposes.
* @param knownSize - known size of the data in the input stream. Pass `-1` value if size is unknown.
*/
Part(const Headers& headers,
const std::shared_ptr<data::stream::InputStream>& inputStream,
const oatpp::String inMemoryData,
data::v_io_size knownSize);
/**
* Constructor.
* @param headers - headers of the part.
*/
Part(const Headers& headers);
/**
* Set part data info.
* @param inputStream - input stream of the part data.
* @param inMemoryData - possible in-memory data of the part. Same data as the referred by input stream. For convenience purposes.
* @param knownSize - known size of the data in the input stream. Pass `-1` value if size is unknown.
*/
void setDataInfo(const std::shared_ptr<data::stream::InputStream>& inputStream,
const oatpp::String inMemoryData,
data::v_io_size knownSize);
/**
* Get name of the part.
* @return - name of the part.
*/
oatpp::String getName() const;
/**
* Get filename of the part (if applicable).
* @return - filename.
*/
oatpp::String getFilename() const;
/**
* Get request's headers map
* @return Headers map
*/
const Headers& getHeaders() const;
/**
* Get header value
* @param headerName
* @return header value
*/
oatpp::String getHeader(const oatpp::data::share::StringKeyLabelCI_FAST& headerName) const;
/**
* Get input stream of the part data.
* @return - input stream of the part data.
*/
std::shared_ptr<data::stream::InputStream> getInputStream() const;
/**
* Get in-memory data (if applicable). <br>
* It may be possible set for the part in case of storing part data in memory. <br>
* This property is optional. Preferred way to access data of the part is through `getInputStream()` method.
* @return - in-memory data.
*/
oatpp::String getInMemoryData() const;
/**
* Return known size of the part data.
* @return - known size of the part data. `-1` - if size is unknown.
*/
data::v_io_size getKnownSize() const;
};
}}}}
#endif // oatpp_web_mime_multipart_Part_hpp

View File

@ -28,9 +28,42 @@
#include "oatpp/core/parser/Caret.hpp"
namespace oatpp { namespace web { namespace mime { namespace multipart {
oatpp::String StatefulParser::parsePartName(p_char8 data, v_int32 size) {
parser::Caret caret(data, size);
if(caret.findText((p_char8)"name=", 5)) {
caret.inc(5);
parser::Caret::Label nameLabel(nullptr);
if(caret.isAtChar('"')) {
nameLabel = caret.parseStringEnclosed('"', '"', '\\');
} else if(caret.isAtChar('\'')) {
nameLabel = caret.parseStringEnclosed('\'', '\'', '\\');
} else {
nameLabel = caret.putLabel();
caret.findCharFromSet(" \t\n\r\f");
nameLabel.end();
}
if(nameLabel) {
return nameLabel.toString();
} else {
throw std::runtime_error("[oatpp::web::mime::multipart::StatefulParser::parsePartName()]: Error. Can't parse part name.");
}
} else {
throw std::runtime_error("[oatpp::web::mime::multipart::StatefulParser::parsePartName()]: Error. Part name is missing.");
}
}
StatefulParser::StatefulParser(const oatpp::String& boundary, const std::shared_ptr<Listener>& listener)
: m_state(STATE_BOUNDARY)
, m_currPartIndex(0)
@ -52,37 +85,10 @@ void StatefulParser::onPartHeaders(const Headers& partHeaders) {
auto it = partHeaders.find("Content-Disposition");
if(it != partHeaders.end()) {
parser::Caret caret(it->second.toString());
m_currPartName = parsePartName(it->second.getData(), it->second.getSize());
if(caret.findText((p_char8)"name=", 5)) {
caret.inc(5);
parser::Caret::Label nameLabel(nullptr);
if(caret.isAtChar('"')) {
nameLabel = caret.parseStringEnclosed('"', '"', '\\');
} else if(caret.isAtChar('\'')) {
nameLabel = caret.parseStringEnclosed('\'', '\'', '\\');
} else {
nameLabel = caret.putLabel();
caret.findCharFromSet(" \t\n\r\f");
nameLabel.end();
}
if(nameLabel) {
m_currPartName = nameLabel.toString();
if(m_listener) {
m_listener->onPartHeaders(m_currPartName, partHeaders);
}
} else {
throw std::runtime_error("[oatpp::web::mime::multipart::StatefulParser::onPartHeaders()]: Error. Can't parse part name.");
}
} else {
throw std::runtime_error("[oatpp::web::mime::multipart::StatefulParser::onPartHeaders()]: Error. Part name is missing.");
if(m_listener) {
m_listener->onPartHeaders(m_currPartName, partHeaders);
}
} else {
@ -138,11 +144,12 @@ v_int32 StatefulParser::parseNext_Boundary(p_char8 data, v_int32 size) {
if(m_currBoundaryCharIndex > 0) {
onPartData(sampleData, m_currBoundaryCharIndex);
} else {
m_checkForBoundary = false;
}
m_state = STATE_DATA;
m_currBoundaryCharIndex = 0;
m_checkForBoundary = false;
return 0;
@ -167,14 +174,16 @@ v_int32 StatefulParser::parseNext_AfterBoundary(p_char8 data, v_int32 size) {
if(size > 1 || m_currBoundaryCharIndex == 1) {
if (m_finishingBoundary && data[1 - m_currBoundaryCharIndex] == '-') {
auto result = 2 - m_currBoundaryCharIndex;
m_state = STATE_DONE;
m_currBoundaryCharIndex = 0;
return 2 - m_currBoundaryCharIndex;
return result;
} else if (!m_finishingBoundary && data[1 - m_currBoundaryCharIndex] == '\n') {
auto result = 2 - m_currBoundaryCharIndex;
m_state = STATE_HEADERS;
m_currBoundaryCharIndex = 0;
m_headerSectionEndAccumulator = 0;
return 2 - m_currBoundaryCharIndex;
return result;
} else {
throw std::runtime_error("[oatpp::web::mime::multipart::StatefulParser::parseNext_AfterBoundary()]: Error. Invalid trailing char.");
}

View File

@ -52,6 +52,11 @@ private:
* `std::unordered_map` of &id:oatpp::data::share::StringKeyLabelCI_FAST; and &id:oatpp::data::share::StringKeyLabel;.
*/
typedef std::unordered_map<oatpp::data::share::StringKeyLabelCI_FAST, oatpp::data::share::StringKeyLabel> Headers;
private:
/**
* Parse name of the part from `Content-Disposition` header.
*/
static oatpp::String parsePartName(p_char8 data, v_int32 size);
public:
/**

View File

@ -24,24 +24,31 @@
#include "StatefulParserTest.hpp"
#include "oatpp/web/mime/multipart/Part.hpp"
#include "oatpp/web/mime/multipart/StatefulParser.hpp"
#include "oatpp/core/data/stream/BufferInputStream.hpp"
#include <unordered_map>
namespace oatpp { namespace test { namespace web { namespace mime { namespace multipart {
namespace {
typedef oatpp::web::mime::multipart::Part Part;
static const char* TEST_DATA_1 =
"--12345\r\n"
"Content-Disposition: form-data; name=\"part1\"\r\n"
"\r\n"
"part1-value\r\n"
"--12345\r\n"
"Content-Disposition: form-data; name=\"part2\" filename=\"filename.txt\"\r\n"
"Content-Disposition: form-data; name='part2' filename=\"filename.txt\"\r\n"
"\r\n"
"--part2-file-content-line1\r\n"
"--1234part2-file-content-line2\r\n"
"--12345\r\n"
"Content-Disposition: form-data; name=\"part3\" filename=\"filename.jpg\"\r\n"
"Content-Disposition: form-data; name=part3 filename=\"filename.jpg\"\r\n"
"\r\n"
"part3-file-binary-data\r\n"
"--12345--\r\n"
@ -52,11 +59,11 @@ namespace {
oatpp::data::stream::ChunkedBuffer m_buffer;
public:
std::unordered_map<oatpp::String, std::shared_ptr<Part>> parts;
void onPartHeaders(const oatpp::String& name, const Headers& partHeaders) override {
OATPP_LOGD("aaa", "part='%s' headers:", name->getData());
for(auto& pair : partHeaders) {
OATPP_LOGD("Header", "name='%s', value='%s'", pair.first.toString()->getData(), pair.second.toString()->getData());
}
auto part = std::make_shared<Part>(partHeaders);
parts.insert({name, part});
}
void onPartData(const oatpp::String& name, p_char8 data, oatpp::data::v_io_size size) override {
@ -66,36 +73,89 @@ namespace {
} else {
auto data = m_buffer.toString();
m_buffer.clear();
OATPP_LOGD("aaa", "part='%s', data='%s'", name->getData(), data->getData());
OATPP_LOGW("aaa", "part end.");
auto& part = parts[name];
OATPP_ASSERT(part);
auto stream = std::make_shared<oatpp::data::stream::BufferInputStream>(data.getPtr(), data->getData(), data->getSize());
part->setDataInfo(stream, data, data->getSize());
}
}
};
void parseStepByStep(const oatpp::String& text,
const oatpp::String& boundary,
const std::shared_ptr<Listener>& listener,
v_int32 step)
{
oatpp::web::mime::multipart::StatefulParser parser(boundary, listener);
oatpp::data::stream::BufferInputStream stream(text.getPtr(), text->getData(), text->getSize());
v_char8 buffer[step];
v_int32 size;
while((size = stream.read(buffer, step)) != 0) {
parser.parseNext(buffer, size);
}
OATPP_ASSERT(parser.finished());
}
void assertPartData(const std::shared_ptr<Part>& part, const oatpp::String& value) {
OATPP_ASSERT(part->getInMemoryData());
OATPP_ASSERT(part->getInMemoryData() == value);
v_int32 bufferSize = 16;
v_char8 buffer[bufferSize];
auto stream = oatpp::data::stream::ChunkedBuffer::createShared();
oatpp::data::stream::transfer(part->getInputStream(), stream, 0, buffer, bufferSize);
oatpp::String readData = stream->toString();
OATPP_ASSERT(readData == part->getInMemoryData());
}
}
void StatefulParserTest::onRun() {
oatpp::String text = TEST_DATA_1;
{
oatpp::web::mime::multipart::StatefulParser parser("12345", std::make_shared<Listener>());
for(v_int32 i = 1; i < text->getSize(); i++) {
for (v_int32 i = 0; i < text->getSize(); i++) {
parser.parseNext(&text->getData()[i], 1);
auto listener = std::make_shared<Listener>();
parseStepByStep(text, "12345", listener, i);
auto& parts = listener->parts;
if(parts.size() != 3) {
OATPP_LOGD(TAG, "TEST_DATA_1 itearation %d", i);
}
OATPP_ASSERT(parts.size() == 3);
auto part1 = parts["part1"];
auto part2 = parts["part2"];
auto part3 = parts["part3"];
OATPP_ASSERT(part1);
OATPP_ASSERT(part2);
OATPP_ASSERT(part3);
OATPP_ASSERT(part1->getFilename().get() == nullptr);
OATPP_ASSERT(part2->getFilename() == "filename.txt");
OATPP_ASSERT(part3->getFilename() == "filename.jpg");
assertPartData(part1, "part1-value");
assertPartData(part2, "--part2-file-content-line1\r\n--1234part2-file-content-line2");
assertPartData(part3, "part3-file-binary-data");
}
OATPP_LOGI(TAG, "Test2.................................................");
{
oatpp::web::mime::multipart::StatefulParser parser("12345", std::make_shared<Listener>());
parser.parseNext(text->getData(), text->getSize());
}
}
}}}}}