Added EE::System::Process class (still a WIP, basic functionality working).

Moved json and subprocess libraries to the eepp thirdparty folder.
Minor improvement to C++ syntax definition.
ecode: FormatterPlugin and LinterPlugin now use the new System::Process class.
This commit is contained in:
Martín Lucas Golini
2022-10-24 02:00:40 -03:00
parent d787887982
commit bc9ea14969
11 changed files with 495 additions and 43 deletions

View File

@@ -39,12 +39,26 @@ class EE_API MemoryManager {
template <class T> static T* deletePtr( T* data ) {
delete data;
#if defined( __GNUC__ ) && __GNUC__ >= 12
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wuse-after-free"
return data;
#pragma GCC diagnostic pop
#else
return data;
#endif
}
template <class T> static T* deleteArrayPtr( T* data ) {
delete[] data;
#if defined( __GNUC__ ) && __GNUC__ >= 12
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wuse-after-free"
return data;
#pragma GCC diagnostic pop
#else
return data;
#endif
}
template <class T> static T* free( T* data ) {
@@ -59,9 +73,7 @@ class EE_API MemoryManager {
#endif
}
inline static void* allocate( size_t size ) {
return malloc( size );
}
inline static void* allocate( size_t size ) { return malloc( size ); }
inline static void* reallocate( void* ptr, size_t size ) {
#if defined( __GNUC__ ) && __GNUC__ >= 12

View File

@@ -0,0 +1,206 @@
#ifndef EE_SYSTEM_PROCESS_HPP
#define EE_SYSTEM_PROCESS_HPP
#include <eepp/config.hpp>
#include <eepp/system/mutex.hpp>
#include <functional>
#include <map>
#include <string>
#include <thread>
namespace EE { namespace System {
class EE_API Process {
public:
enum Options {
// stdout and stderr are the same FILE.
CombinedStdoutStderr = 0x1,
// The child process should inherit the environment variables of the parent.
InheritEnvironment = 0x2,
// Enable asynchronous reading of stdout/stderr before it has completed.
EnableAsync = 0x4,
// Enable the child process to be spawned with no window visible if supported
// by the platform.
NoWindow = 0x8,
// Search for program names in the PATH variable. Always enabled on Windows.
// Note: this will **not** search for paths in any provided custom environment
// and instead uses the PATH of the spawning process.
SearchUserPath = 0x10
};
static inline constexpr Uint32 getDefaultOptions() {
return Options::SearchUserPath | Options::InheritEnvironment | Options::NoWindow;
}
struct Config {
size_t bufferSize = 131072;
Uint32 options = getDefaultOptions();
};
typedef std::function<void( const char* bytes, size_t n )> ReadFn;
Process();
/** @brief Create a process.
** @param command Command line to execute for this process.
** @param options A bit field of Options's to pass. */
Process( const std::string& command, const Uint32& options = getDefaultOptions(),
const size_t& bufferSize = 132072 );
~Process();
/** @brief Create a process.
** @param command Command line to execute for this process.
** @param options A bit field of Options's to pass.
** @return On success true is returned. */
bool create( const std::string& command, const Uint32& options = getDefaultOptions(),
const std::map<std::string, std::string>& environment = {} );
/** @brief Starts a new thread to receive all stdout and stderr data */
void startAsyncRead( ReadFn readStdOut = nullptr, ReadFn readStdErr = nullptr );
/** @brief Read all standard output from the child process.
** @param buffer The buffer to read into.
** @return The number of bytes actually read into buffer. Can only be 0 if the
** process has complete.
**
** The only safe way to read from the standard output of a process during it's
** execution is to use the `Option::EnableAsync` option in
** conjuction with this method. */
size_t readAllStdOut( std::string& buffer );
/** @brief Read the standard output from the child process.
** @param buffer The buffer to read into.
** @return The number of bytes actually read into buffer. Can only be 0 if the
** process has complete.
**
** The only safe way to read from the standard output of a process during it's
** execution is to use the `Option::EnableAsync` option in
** conjuction with this method. */
size_t readStdOut( std::string& buffer );
/** @brief Read the standard output from the child process.
** @param buffer The buffer to read into.
** @param size The maximum number of bytes to read.
** @return The number of bytes actually read into buffer. Can only be 0 if the
** process has complete.
**
** The only safe way to read from the standard output of a process during it's
** execution is to use the `Option::EnableAsync` option in
** conjuction with this method. */
size_t readStdOut( char* const buffer, const size_t& size );
/** @brief Read all the standard error from the child process.
** @param buffer The buffer to read into.
** @return The number of bytes actually read into buffer. Can only be 0 if the
** process has complete.
**
** The only safe way to read from the standard error of a process during it's
** execution is to use the `Option::EnableAsync` option in
** conjuction with this method. */
size_t readAllStdErr( std::string& buffer );
/** @brief Read the standard error from the child process.
** @param buffer The buffer to read into.
** @return The number of bytes actually read into buffer. Can only be 0 if the
** process has complete.
**
** The only safe way to read from the standard error of a process during it's
** execution is to use the `Option::EnableAsync` option in
** conjuction with this method. */
size_t readStdErr( std::string& buffer );
/** @brief Read the standard error from the child process.
** @param buffer The buffer to read into.
** @param size The maximum number of bytes to read.
** @return The number of bytes actually read into buffer. Can only be 0 if the
** process has complete.
**
** The only safe way to read from the standard error of a process during it's
** execution is to use the `Option::EnableAsync` option in
** conjuction with this method. */
size_t readStdErr( char* const buffer, const size_t& size );
/** @brief Write the standard output from the child process.
** @param buffer The buffer to write into.
** @param size The number of bytes to write.
** @return The number of bytes actually written into buffer. */
size_t write( const char* buffer, const size_t& size );
/** @brief Write the standard output from the child process.
** @param buffer The buffer to write into.
** @return The number of bytes actually written into buffer. */
size_t write( const std::string& buffer );
/** @brief Wait for a process to finish execution.
** @param returnCodeOut The return code of the returned process (can be nullptr).
** @return On success true is returned.
**
** Joining a process will close the stdin pipe to the process. */
bool join( int* const returnCodeOut );
/** @brief Terminate a previously created process.
** @return On success true is returned.
**
** If the process to be destroyed had not finished execution, it will be
** terminated (i.e killed). */
bool kill();
/** @brief Destroy a previously created process.
** @return On success true is returned.
**
** If the process to be destroyed had not finished execution, it may out live
** the parent process. */
bool destroy();
/** @brief Returns if the subprocess is currently still alive and executing.
** @return If the process is still alive true returned. */
bool isAlive();
/** @brief Get the standard input file for a process.
** @return The file for standard input of the process.
**
** The file returned can be written to by the parent process to feed data to
** the standard input of the process. */
FILE* getStdIn() const;
/** @brief Get the standard output file for a process.
** @return The file for standard output of the process.
**
** The file returned can be read from by the parent process to read data from
** the standard output of the child process. */
FILE* getStdOut() const;
/** @brief Get the standard error file for a process.
** @return The file for standard error of the process.
**
** The file returned can be read from by the parent process to read data from
** the standard error of the child process.
**
** If the process was created with the Option::CombinedStdoutStderr
** option bit set, this function will return NULL, and the getStdOut
** function should be used for both the standard output and error combined. */
FILE* getStdErr() const;
/** Indicates that the process must start its shutdown */
void startShutdown();
protected:
void* mProcess{ nullptr };
bool mShuttingDown{ false };
bool mIsAsync{ false };
size_t mBufferSize{ 131072 };
std::thread mStdOutThread;
std::thread mStdErrThread;
Mutex mStdInMutex;
ReadFn mReadStdOutFn;
ReadFn mReadStdErrFn;
};
}} // namespace EE::System
#endif // EE_SYSTEM_PROCESS_HPP

View File

@@ -49,4 +49,4 @@ then
chmod +x "$APPIMAGETOOL"
fi
$APPIMAGETOOL ecode.app ecode-"$ECODE_MAJOR_VERSION"."$ECODE_MINOR_VERSION"."$ECODE_PATCH_LEVEL"-"$(arch)".AppImage
$APPIMAGETOOL ecode.app ecode-linux-"$ECODE_MAJOR_VERSION"."$ECODE_MINOR_VERSION"."$ECODE_PATCH_LEVEL"-"$(arch)".AppImage

View File

@@ -257,6 +257,7 @@
../../include/eepp/system/pack.hpp
../../include/eepp/system/packmanager.hpp
../../include/eepp/system/pak.hpp
../../include/eepp/system/process.hpp
../../include/eepp/system/rc4.hpp
../../include/eepp/system/resourceloader.hpp
../../include/eepp/system/resourcemanager.hpp
@@ -781,6 +782,7 @@
../../src/eepp/system/platform/win/threadimpl.hpp
../../src/eepp/system/platform/win/threadlocalimpl.cpp
../../src/eepp/system/platform/win/threadlocalimpl.hpp
../../src/eepp/system/process.cpp
../../src/eepp/system/rc4.cpp
../../src/eepp/system/resourceloader.cpp
../../src/eepp/system/sys.cpp

250
src/eepp/system/process.cpp Normal file
View File

@@ -0,0 +1,250 @@
#include <bitset>
#include <eepp/core/debug.hpp>
#include <eepp/core/memorymanager.hpp>
#include <eepp/core/string.hpp>
#include <eepp/system/lock.hpp>
#include <eepp/system/process.hpp>
#include <thirdparty/subprocess/subprocess.h>
#include <vector>
#ifdef EE_PLATFORM_POSIX
#include <fcntl.h>
#include <poll.h>
#include <stdio.h>
#elif EE_PLATFORM == EE_PLATFORM_WIN
#include <windows.h>
#endif
namespace EE { namespace System {
#define PROCESS_PTR ( (struct subprocess_s*)mProcess )
Process::Process() {}
Process::Process( const std::string& command, const Uint32& options, const size_t& bufferSize ) :
mBufferSize( bufferSize ) {
create( command, options );
}
Process::~Process() {
if ( mStdOutThread.joinable() )
mStdOutThread.join();
if ( mStdErrThread.joinable() )
mStdErrThread.join();
destroy();
eeFree( mProcess );
}
bool Process::create( const std::string& command, const Uint32& options,
const std::map<std::string, std::string>& environment ) {
if ( mProcess )
return false;
std::vector<std::string> cmdArr = String::split( command, " ", "", "\"", true );
std::vector<const char*> strings;
for ( size_t i = 0; i < cmdArr.size(); ++i )
strings.push_back( cmdArr[i].c_str() );
strings.push_back( NULL );
mProcess = eeMalloc( sizeof( subprocess_s ) );
memset( mProcess, 0, sizeof( subprocess_s ) );
if ( !environment.empty() ) {
std::vector<std::string> envArr;
std::vector<const char*> envStrings;
for ( const auto& pair : environment ) {
envArr.push_back( String::format( "%s=%s", pair.first.c_str(), pair.second.c_str() ) );
envStrings.push_back( envArr[envArr.size() - 1].c_str() );
}
envStrings.push_back( NULL );
return 0 == subprocess_create_ex( strings.data(), options, envStrings.data(), PROCESS_PTR );
}
return 0 == subprocess_create( strings.data(), options, PROCESS_PTR );
}
size_t Process::readAllStdOut( std::string& buffer ) {
size_t bytesRead = 0;
size_t totalBytesRead = 0;
const size_t chunkSize = mBufferSize;
totalBytesRead = bytesRead = readStdOut( (char* const)buffer.c_str(), buffer.size() );
while ( bytesRead != 0 && isAlive() && !mShuttingDown ) {
bytesRead = readStdOut( (char* const)buffer.c_str() + bytesRead, chunkSize );
if ( bytesRead ) {
totalBytesRead += bytesRead;
buffer.resize( totalBytesRead + chunkSize );
}
}
return totalBytesRead;
}
size_t Process::readStdOut( std::string& buffer ) {
return readStdOut( (char* const)buffer.c_str(), buffer.size() );
}
size_t Process::readStdOut( char* const buffer, const size_t& size ) {
eeASSERT( mProcess != nullptr );
return subprocess_read_stdout( PROCESS_PTR, buffer, size );
}
size_t Process::readAllStdErr( std::string& buffer ) {
size_t bytesRead = 0;
size_t totalBytesRead = 0;
const size_t chunkSize = 4096;
totalBytesRead = bytesRead = readStdErr( (char* const)buffer.c_str(), buffer.size() );
while ( bytesRead != 0 && isAlive() && !mShuttingDown ) {
bytesRead = readStdErr( (char* const)buffer.c_str() + bytesRead, chunkSize );
if ( bytesRead ) {
totalBytesRead += bytesRead;
buffer.resize( totalBytesRead + chunkSize );
}
}
return totalBytesRead;
}
size_t Process::readStdErr( std::string& buffer ) {
return readStdErr( (char* const)buffer.c_str(), buffer.size() );
}
size_t Process::readStdErr( char* const buffer, const size_t& size ) {
eeASSERT( mProcess != nullptr );
return subprocess_read_stderr( PROCESS_PTR, buffer, size );
}
size_t Process::write( const char* buffer, const size_t& size ) {
eeASSERT( mProcess != nullptr );
Lock l( mStdInMutex );
FILE* stdInFile = subprocess_stdin( PROCESS_PTR );
return fwrite( buffer, sizeof( char ), size, stdInFile );
}
size_t Process::write( const std::string& buffer ) {
return write( buffer.c_str(), buffer.size() );
}
bool Process::join( int* const returnCodeOut ) {
eeASSERT( mProcess != nullptr );
return 0 == subprocess_join( PROCESS_PTR, returnCodeOut );
}
bool Process::kill() {
eeASSERT( mProcess != nullptr );
return 0 == subprocess_terminate( PROCESS_PTR );
}
bool Process::destroy() {
eeASSERT( mProcess != nullptr );
return 0 == subprocess_destroy( PROCESS_PTR );
}
bool Process::isAlive() {
eeASSERT( mProcess != nullptr );
return 0 != subprocess_alive( PROCESS_PTR );
}
FILE* Process::getStdIn() const {
eeASSERT( mProcess != nullptr );
return subprocess_stdin( PROCESS_PTR );
}
FILE* Process::getStdOut() const {
eeASSERT( mProcess != nullptr );
return subprocess_stdout( PROCESS_PTR );
}
FILE* Process::getStdErr() const {
eeASSERT( mProcess != nullptr );
return subprocess_stderr( PROCESS_PTR );
}
void Process::startShutdown() {
mShuttingDown = true;
}
void Process::startAsyncRead( ReadFn readStdOut, ReadFn readStdErr ) {
eeASSERT( mProcess != nullptr );
mReadStdOutFn = readStdOut;
mReadStdErrFn = readStdErr;
#if EE_PLATFORM == EE_PLATFORM_WIN
// TODO: Implement WaitForMultipleObjectsEx
void* stdOutFd =
SUBPROCESS_PTR_CAST( void*, _get_osfhandle( _fileno( PROCESS_PTR->stdout_file ) ) );
void* stdErrFd =
SUBPROCESS_PTR_CAST( void*, _get_osfhandle( _fileno( PROCESS_PTR->stderr_file ) ) );
if ( stdOutFd ) {
mStdOutThread = std::thread( [this, stdOutFd]() {
DWORD n;
std::unique_ptr<char[]> buffer( new char[mBufferSize] );
while ( !mShuttingDown ) {
BOOL bSuccess = ReadFile( stdOutFd, static_cast<CHAR*>( buffer.get() ),
static_cast<DWORD>( mBufferSize ), &n, nullptr );
if ( !bSuccess || n == 0 )
break;
mReadStdOutFn( buffer.get(), static_cast<size_t>( n ) );
}
} );
}
if ( stdErrFd ) {
mStdErrThread = std::thread( [this, stdErrFd]() {
DWORD n;
std::unique_ptr<char[]> buffer( new char[mBufferSize] );
while ( !mShuttingDown ) {
BOOL bSuccess = ReadFile( stdErrFd, static_cast<CHAR*>( buffer.get() ),
static_cast<DWORD>( mBufferSize ), &n, nullptr );
if ( !bSuccess || n == 0 )
break;
mReadStdErrFn( buffer.get(), static_cast<size_t>( n ) );
}
} );
}
#elif defined( EE_PLATFORM_POSIX )
mStdOutThread = std::thread( [this] {
auto stdOutFd = fileno( PROCESS_PTR->stdout_file );
auto stdErrFd = PROCESS_PTR->stderr_file ? fileno( PROCESS_PTR->stderr_file ) : 0;
std::vector<pollfd> pollfds;
std::bitset<2> fdIsStdOut;
if ( stdOutFd ) {
fdIsStdOut.set( pollfds.size() );
pollfds.emplace_back();
pollfds.back().fd =
fcntl( stdOutFd, F_SETFL, fcntl( stdOutFd, F_GETFL ) | O_NONBLOCK ) == 0 ? stdOutFd
: -1;
pollfds.back().events = POLLIN;
}
if ( stdErrFd ) {
pollfds.emplace_back();
pollfds.back().fd =
fcntl( stdErrFd, F_SETFL, fcntl( stdErrFd, F_GETFL ) | O_NONBLOCK ) == 0 ? stdErrFd
: -1;
pollfds.back().events = POLLIN;
}
auto buffer = std::unique_ptr<char[]>( new char[mBufferSize] );
bool anyOpen = !pollfds.empty();
while ( anyOpen && !mShuttingDown &&
( poll( pollfds.data(), static_cast<nfds_t>( pollfds.size() ), -1 ) > 0 ||
errno == EINTR ) ) {
anyOpen = false;
for ( size_t i = 0; i < pollfds.size(); ++i ) {
if ( pollfds[i].fd >= 0 ) {
if ( pollfds[i].revents & POLLIN ) {
const ssize_t n = read( pollfds[i].fd, buffer.get(), mBufferSize );
if ( n > 0 ) {
if ( fdIsStdOut[i] )
mReadStdOutFn( buffer.get(), static_cast<size_t>( n ) );
else
mReadStdErrFn( buffer.get(), static_cast<size_t>( n ) );
} else if ( n < 0 && errno != EINTR && errno != EAGAIN &&
errno != EWOULDBLOCK ) {
pollfds[i].fd = -1;
continue;
}
}
if ( pollfds[i].revents & ( POLLERR | POLLHUP | POLLNVAL ) ) {
pollfds[i].fd = -1;
continue;
}
anyOpen = true;
}
}
}
} );
#endif
}
}} // namespace EE::System

View File

@@ -694,7 +694,9 @@ void SyntaxDefinitionManager::addCPP() {
{ { "R%\"xml%(", "%)xml%\"" }, "function", "XML" },
{ { "R%\"css%(", "%)css%\"" }, "function", "CSS" },
{ { "R%\"html%(", "%)html%\"" }, "function", "HTML" },
{ { "R%\"json%(", "%)json%\"" }, "function", "JSON" },
{ { "R\"[%a-\"]+%(", "%)[%a-\"]+%\"" }, "string" },
{ { "R\"%(", "%)\"" }, "string" },
{ { "//.-\n" }, "comment" },
{ { "/%*", "%*/" }, "comment" },
{ { "\"", "\"", "\\" }, "string" },

View File

@@ -1,12 +1,12 @@
#include "appconfig.hpp"
#include "ecode.hpp"
#include "plugins/pluginmanager.hpp"
#include "thirdparty/json.hpp"
#include <eepp/network/uri.hpp>
#include <eepp/system/filesystem.hpp>
#include <eepp/system/md5.hpp>
#include <eepp/system/sys.hpp>
#include <eterm/ui/uiterminal.hpp>
#include <nlohmann/json.hpp>
using namespace EE::Network;
using namespace eterm::UI;

View File

@@ -1,13 +1,13 @@
#include "formatterplugin.hpp"
#include "../../scopedop.hpp"
#include "../../thirdparty/json.hpp"
#include "../../thirdparty/subprocess.h"
#include <eepp/system/filesystem.hpp>
#include <eepp/system/iostreamstring.hpp>
#include <eepp/system/lock.hpp>
#include <eepp/system/luapattern.hpp>
#include <eepp/system/process.hpp>
#include <eepp/ui/css/stylesheet.hpp>
#include <eepp/ui/css/stylesheetparser.hpp>
#include <nlohmann/json.hpp>
#include <random>
#define PUGIXML_HEADER_ONLY
#include <pugixml/pugixml.hpp>
@@ -294,34 +294,24 @@ void FormatterPlugin::runFormatter( UICodeEditor* editor, const Formatter& forma
std::string cmd( formatter.command );
String::replaceAll( cmd, "$FILENAME", "\"" + path + "\"" );
std::vector<std::string> cmdArr = String::split( cmd, " ", "", "\"", true );
std::vector<const char*> strings;
for ( size_t i = 0; i < cmdArr.size(); ++i )
strings.push_back( cmdArr[i].c_str() );
strings.push_back( NULL );
struct subprocess_s subprocess;
int result =
subprocess_create( strings.data(),
subprocess_option_search_user_path |
subprocess_option_inherit_environment | subprocess_option_no_window,
&subprocess );
if ( 0 == result ) {
Process process;
if ( process.create( cmd ) ) {
std::string buffer( 1024, '\0' );
std::string data;
unsigned bytesRead = 0;
int returnCode;
do {
bytesRead = subprocess_read_stdout( &subprocess, &buffer[0], buffer.size() );
bytesRead = process.readStdOut( buffer );
data += buffer.substr( 0, bytesRead );
} while ( bytesRead != 0 && subprocess_alive( &subprocess ) && !mShuttingDown );
} while ( bytesRead != 0 && process.isAlive() && !mShuttingDown );
if ( mShuttingDown ) {
subprocess_terminate( &subprocess );
process.kill();
return;
}
subprocess_join( &subprocess, &returnCode );
subprocess_destroy( &subprocess );
process.join( &returnCode );
process.destroy();
// Log::info( "Formatter result:\n%s", data.c_str() );

View File

@@ -1,7 +1,5 @@
#include "linterplugin.hpp"
#include "../../scopedop.hpp"
#include "../../thirdparty/json.hpp"
#include "../../thirdparty/subprocess.h"
#include <algorithm>
#include <eepp/graphics/primitives.hpp>
#include <eepp/graphics/text.hpp>
@@ -9,7 +7,9 @@
#include <eepp/system/iostreamstring.hpp>
#include <eepp/system/lock.hpp>
#include <eepp/system/luapattern.hpp>
#include <eepp/system/process.hpp>
#include <eepp/ui/uitooltip.hpp>
#include <nlohmann/json.hpp>
#include <random>
using json = nlohmann::json;
@@ -321,34 +321,24 @@ void LinterPlugin::runLinter( std::shared_ptr<TextDocument> doc, const Linter& l
Clock clock;
std::string cmd( linter.command );
String::replaceAll( cmd, "$FILENAME", "\"" + path + "\"" );
std::vector<std::string> cmdArr = String::split( cmd, " ", "", "\"", true );
std::vector<const char*> strings;
for ( size_t i = 0; i < cmdArr.size(); ++i )
strings.push_back( cmdArr[i].c_str() );
strings.push_back( NULL );
struct subprocess_s subprocess;
int result = subprocess_create(
strings.data(),
subprocess_option_search_user_path | subprocess_option_inherit_environment |
subprocess_option_combined_stdout_stderr | subprocess_option_no_window,
&subprocess );
if ( 0 == result ) {
Process process;
if ( process.create( cmd, Process::getDefaultOptions() | Process::CombinedStdoutStderr ) ) {
std::string buffer( 1024, '\0' );
std::string data;
unsigned bytesRead = 0;
int returnCode;
do {
bytesRead = subprocess_read_stdout( &subprocess, &buffer[0], buffer.size() );
bytesRead = process.readStdOut( buffer );
data += buffer.substr( 0, bytesRead );
} while ( bytesRead != 0 && subprocess_alive( &subprocess ) && !mShuttingDown );
} while ( bytesRead != 0 && process.isAlive() && !mShuttingDown );
if ( mShuttingDown ) {
subprocess_terminate( &subprocess );
process.kill();
return;
}
subprocess_join( &subprocess, &returnCode );
subprocess_destroy( &subprocess );
process.join( &returnCode );
process.destroy();
if ( linter.hasNoErrorsExitCode && linter.noErrorsExitCode == returnCode ) {
Lock matchesLock( mMatchesMutex );