File: /volume1/@appstore/HyperBackup/addon/aws_s3/php/v3/agent_server_s3.php
<?php
if (count(get_included_files()) == 1) {
define('__name__', "__main__");
} else {
define('__name__', "x");
}
function log_debug()
{
$args = func_get_args();
$fmt = $args[0];
list($b, $b2) = debug_backtrace();
$bt = basename($b["file"]).":".$b["line"].":".$b2["function"].": ";
$args[0] = $bt;
return vfprintf(STDERR, "%s".$fmt."\n", $args);
}
function log_debug_r($mix)
{
ob_start();
print_r($mix);
$debug_msg = ob_get_clean();
list($b, $b2) = debug_backtrace();
$bt = basename($b["file"]).":".$b["line"].":".$b2["function"].": ";
return fprintf(STDERR, "%s[%s]\n", $bt, $debug_msg);
}
function getPath($fpath)
{
$sdk_dir = getenv("AWS_SDK_DIR");
if (FALSE == $sdk_dir) {
$sdk_dir = "/usr/syno/aws";
}
return $sdk_dir."/".$fpath;
}
if (false) {
$lib_prefix = '/mnt/source/libsynobackup/lib/s3common/aws-sdk';
define('AWS_PHAR', true);
require_once $lib_prefix.'/aws-autoloader.php';
} else {
$aws_sdk_path = getenv("AWS_SDK_PATH");
if (FALSE == $aws_sdk_path) {
require getPath('v3/aws.phar');
} else {
require $aws_sdk_path;
}
}
use Aws\Middleware;
use Aws\S3\Exception\S3Exception;
use Aws\Handler\GuzzleV6\GuzzleHandler;
use Psr\Http\Message\RequestInterface;
use Aws\ResultInterface;
use GuzzleHttp\Exception\RequestException;
// because AWS time stamp use UTC time
date_default_timezone_set("UTC");
error_reporting(0);
function _convert_exception($e)
{
try {
log_debug($e);
} catch(Exception $ee) {
; // pass
}
if (is_subclass_of($e, 'Aws\Exception\AwsException')) {
// http://docs.aws.amazon.com/AmazonS3/latest/API/ErrorResponses.html
$errMsg = $e->getMessage();
$ret = array("success" => false, "error_class" => get_class($e), "error_message" => $errMsg);
$httpStatusCode = $e->getStatusCode();
if ($httpStatusCode === null) {
$pos = strpos($errMsg, "cURL error");
if ($pos !== false && 1 === sscanf(substr($errMsg, $pos), "cURL error %d", $curlErrno)) {
$ret["curl_error_code"] = $curlErrno;
}
} else {
$ret["http_status_code"] = $httpStatusCode;
$ret["aws_error_type"] = $e->getAwsErrorType();
$ret["aws_error_code"] = $e->getAwsErrorCode();
}
return $ret;
} elseif (is_subclass_of($e, 'GuzzleHttp\Exception\RequestException')) {
$ctx = $e->getHandlerContext();
return array(
"success" => false,
"error_class" => get_class($e),
"error_message" => $e->getMessage(),
"curl_message" => $ctx['error'],
"curl_error_code" => $ctx['errno']
);
} else {
return array(
"success" => false,
"error_class" => get_class($e),
"error_message" => $e->getMessage(),
);
}
}
function _convert_time($time_str)
{
// 2014-06-17T10:30:51.000Z
$time_str = substr($time_str, 0, 19);
$ts = strptime($time_str, '%FT%T');
$tt = mktime($ts['tm_hour'], $ts['tm_min'], $ts['tm_sec'],
$ts['tm_mon']+1, $ts['tm_mday'], ($ts['tm_year'] + 1900));
return $tt;
}
class ServerTerminateException extends Exception {};
class SynoCommandPipeIO
{
function read_int()
{
$n = 4;
while ($n > 0) {
if (false == ($read_data = fread(STDIN, $n))) {
if (feof(STDIN)) {
throw new ServerTerminateException();
}
throw new Exception("read failed");
}
$data .= $read_data;
$n -= mb_strlen($read_data, '8bit');
}
if ("" === $data && feof(STDIN)) {
throw new ServerTerminateException();
}
$unpack_data = unpack("I", $data);
return $unpack_data[1];
}
function read_string()
{
$len = $this->read_int();
if (0 === $len) {
return "";
}
$n = $len;
while ($n > 0) {
$to_read = 4096 > $n ? $n : 4096;
if (FALSE == ($read_str = fread(STDIN, $to_read))) {
throw new Exception("read failed");
}
$str .= $read_str;
$n -= mb_strlen($read_str, '8bit');
if (feof(STDIN) && $n > 0) {
throw new Exception("read eof");
}
}
return $str;
}
function read_json()
{
$str = $this->read_string();
return json_decode($str, true);
}
function write_int($val)
{
$pack_data = pack("I", $val);
fwrite(STDOUT, $pack_data, 4);
//fprintf(STDERR, "[%d] write: %d\n", posix_getpid(), $val);
}
function write_string($msg)
{
$len = strlen($msg);
$this->write_int($len);
fwrite(STDOUT, $msg, $len);
//fprintf(STDERR, "[%d] write: %s\n", posix_getpid(), $msg);
}
function write_json($json)
{
$this->write_string(json_encode($json));
fflush(STDOUT);
}
function write_exception($e)
{
$this->write_json(_convert_exception($e));
}
}
class SynoS3Wrapper
{
private $client = null;
private $io = null;
function __construct($aws_config, $io)
{
$this->io = $io;
$this->createS3Client($aws_config);
}
function createS3Client($aws_config)
{
log_debug("\033[34minit_client\033[0m");
$this->client = new \Aws\S3\S3MultiRegionClient($aws_config);
}
private $prev_uploaded_size = 0;
private $last_update_time = 0;
function onUploadProgress($expectedDownload, $downloaded, $expectedUpload, $uploaded)
{
if ($this->prev_uploaded_size >= $uploaded) {
// ignore insane and no information case
return ;
}
$this->prev_uploaded_size = $uploaded;
if (!$this->io || time() - $this->last_update_time < 3) {
return ;
}
$this->io->write_json(array(
"success" => true,
"complete" => false,
"uploaded" => $this->prev_uploaded_size,
));
$this->last_update_time = time();
}
function initUploadProgress(&$in_json)
{
$this->prev_uploaded_size = 0;
$this->last_update_time = 0;
$in_json["@http"] = [
'progress' => function($expectedDl, $dl, $expectedUl, $ul) {
call_user_func_array(array($this, 'onUploadProgress'), array($expectedDl, $dl, $expectedUl, $ul));
}
];
}
function getVersion($in_json)
{
return array(
"success" => true,
"version" => Aws\Sdk::VERSION,
);
}
function getRegion()
{
return $this->client->getRegion();
}
function createBucket($in_json)
{
try {
$createBucketConfig = [
'Bucket' => $in_json['Bucket']];
if (!empty($in_json["LocationConstraint"])) {
$createBucketConfig['CreateBucketConfiguration'] =
['LocationConstraint' => $in_json["LocationConstraint"]];
}
$res = $this->client->CreateBucket($createBucketConfig);
} catch (S3Exception $e1) {
// DSM #69035 - workaround for frankfurt problem.
try {
$res = $this->getBucketLocation(array("Bucket" => $in_json["Bucket"]));
} catch (Exception $e2) {
throw($e1);
}
$context = array('response' => new GuzzleHttp\Psr7\Response(409), 'code' => 'BucketAlreadyOwnedByYou');
$new_e = new S3Exception("Your previous request to create the named bucket succeeded and you already own it.", $e1->getCommand(), $context);
throw($new_e);
}
return array("success" => true);
}
function headBucket($in_json)
{
$res = $this->client->HeadBucket($in_json);
return array("success" => true);
}
function listBuckets($in_json)
{
$res = $this->client->ListBuckets($in_json);
$out_json = array(
"success" => true,
"Buckets" => array()
);
foreach ($res->get("Buckets") as $rec) {
$out_json["Buckets"][] = $rec["Name"];
}
return $out_json;
}
function getBucketLocation($in_json)
{
$res = $this->client->getBucketLocation($in_json);
return array(
"success" => true,
"Location" => $res->get("LocationConstraint")
);
}
function putObject($in_json)
{
if (preg_match('/[\x00-\x1F\x7F]/', $in_json["Key"])) {
log_debug("key contains control char. (%s)", $in_json["Key"]);
return array(
"success" => false,
"error_class" => "PhpPathContainCtrlChar",
"error_message" => "path contains control chars"
);
}
if (false === ($handle = fopen($in_json["Body"], 'r'))) {
throw new Exception("can not open file [".$in_json["Body"]."]");
}
$in_json["Body"] = $handle;
try {
$this->initUploadProgress($in_json);
$res = $this->client->PutObject($in_json);
if (is_resource($handle)) {
fclose($handle);
}
return array(
"success" => true,
"RequestId" => $res->get("RequestId"),
"ETag" => $res->get("ETag")
);
} catch (Exception $e) {
if (is_resource($handle)) {
fclose($handle);
}
throw $e;
}
}
function listObjects($in_json)
{
$res = $this->client->ListObjects($in_json);
$prefix_len = strlen($in_json["Prefix"]);
$out_json = array(
"success" => true,
"count" => 0,
);
if ($res->get("NextMarker")) {
$out_json["NextMarker"] = $res->get("NextMarker");
} elseif ($res->get("IsTruncated")) {
$out_json["NextMarker"] = $res->get("Contents")[count($res->get("Contents")) - 1]["Key"];
}
$out_json["folder"] = array();
$out_json["file"] = array();
if ($res->get("CommonPrefixes")) {
foreach ($res->get("CommonPrefixes") as $rec) {
$out_json["folder"][] = array(
"Name" => substr($rec["Prefix"], $prefix_len),
);
$out_json["count"] += 1;
}
}
if ($res->get("Contents")) {
foreach ($res->get("Contents") as $rec) {
if (strlen($rec["Key"]) <= $prefix_len) {
continue;
}
if ('/' === substr($rec["Key"], -1)) {
$out_json["folder"][] = array(
"Name" => substr($rec["Key"], $prefix_len),
);
} else {
$out_json["file"][] = array(
"Name" => substr($rec["Key"], $prefix_len),
"ETag" => $rec["ETag"],
"LastModified" => _convert_time($rec["LastModified"]),
"ContentLength" => $rec["Size"],
);
}
$out_json["count"] += 1;
}
}
return $out_json;
}
function headObject($in_json)
{
$res = $this->client->HeadObject($in_json);
return array(
"success" => true,
"LastModified" => _convert_time($res->get("LastModified")),
"ETag" => $res->get("ETag"),
"ContentLength" => $res->get("ContentLength")
);
}
function deleteObject($in_json)
{
$res = $this->client->DeleteObject($in_json);
return array(
"success" => true,
"RequestId" => $res->get("RequestId"),
);
}
function getObject($in_json)
{
$res = $this->client->GetObject($in_json);
return array(
"success" => true,
"ETag" => $res->get("ETag"),
"RequestId" => $res->get("RequestId")
);
}
function copyObject($in_json)
{
$res = $this->client->CopyObject($in_json);
return array(
"success" => true,
"RequestId" => $res->get("RequestId")
);
}
function createMultipartUpload($in_json)
{
if (preg_match('/[\x00-\x1F\x7F]/', $in_json["Key"])) {
log_debug("key contains control char. (%s)", $in_json["Key"]);
return array(
"success" => false,
"error_class" => "PhpPathContainCtrlChar",
"error_message" => "path contains control chars"
);
}
$res = $this->client->createMultipartUpload($in_json);
return array(
"success" => true,
"UploadId" => $res->get("UploadId")
);
}
function uploadPart($in_json)
{
if (false === ($handle = fopen($in_json["Body"], 'r'))) {
throw new Exception("can not open file [".$in_json["Body"]."]");
}
$in_json["Body"] = $handle;
try {
$this->initUploadProgress($in_json);
$res = $this->client->uploadPart($in_json);
if (is_resource($handle)) {
fclose($handle);
}
return array(
"success" => true,
"RequestId" => $res->get("RequestId"),
"ETag" => $res->get("ETag")
);
} catch (Exception $e) {
if (is_resource($handle)) {
fclose($handle);
}
throw $e;
}
}
function completeMultipartUpload($in_json)
{
$res = $this->client->completeMultipartUpload($in_json);
return array(
"success" => true,
"ETag" => $res->get("ETag"),
"RequestId" => $res->get("RequestId")
);
}
function abortMultipartUpload($in_json)
{
$res = $this->client->abortMultipartUpload($in_json);
return array("success" => true);
}
function listMultipartUploads($in_json)
{
$res = $this->client->listMultipartUploads($in_json);
$out_json = array(
'success' => true,
'uploads' => array(),
);
if (!$res->get("Uploads")) {
return $out_json;
}
foreach ($res->get("Uploads") as $rec) {
$out_json['uploads'][] = array(
"Key" => $rec["Key"],
"UploadId" => $rec["UploadId"],
);
}
return $out_json;
}
function listParts($in_json)
{
$res = $this->client->listParts($in_json);
$out_json = array(
'success' => true,
'parts' => array()
);
if ($res->get("IsTruncated")) {
$out_json['NextPartNumberMarker'] = $res->get('NextPartNumberMarker');
}
if ($res->get("Parts")) {
foreach ($res->get("Parts") as $rec) {
$out_json['parts'][] = array(
'PartNumber' => $rec['PartNumber'],
'LastModified' => $rec['LastModified'],
'ETag' => $rec['ETag'],
'Size' => $rec['Size'],
);
}
}
return $out_json;
}
}
function check_need_redirect($wrapper, $aws_config, $in_json)
{
if (array_key_exists("check_region", $aws_config) && false !== $aws_config["check_region"]) {
// only check once for this process live cycle
return false;
}
if (array_key_exists("endpoint", $aws_config) && false !== $aws_config["endpoint"]) {
return false;
}
if (!array_key_exists("Bucket", $in_json)) {
return false;
}
if (false !== strpos($in_json["fn"], "Bucket")) {
return false;
}
log_debug("\033[33mcheck bucket region\033[0m");
$res = $wrapper->getBucketLocation(array("Bucket" => $in_json["Bucket"]));
if ($res["Location"] === "EU") {
return "eu-west-1";
} else {
return $res["Location"];
}
}
function get_region_from_url($url)
{
$matches = array();
if (1 == preg_match("/(^|\.|\/)s3\.([^.]*)\./i", $url, $matches)) {
return $matches[2];
}
return "";
}
function can_try_region_from_url($aws_config)
{
if (empty(getenv("AWS_REGION"))
&& array_key_exists("endpoint", $aws_config)
&& is_string($aws_config["endpoint"])) {
return !empty(get_region_from_url($aws_config["endpoint"]));
}
return false;
}
function _main()
{
$io = new SynoCommandPipeIO();
try {
$handler = new GuzzleHandler(
new GuzzleHttp\Client([
'curl' => [
CURLOPT_CONNECTTIMEOUT => 60,
CURLOPT_LOW_SPEED_LIMIT => 1,
CURLOPT_LOW_SPEED_TIME => 1800,
CURLOPT_SSLVERSION => CURL_SSLVERSION_TLSv1,
CURLOPT_VERBOSE => getenv("AWS_DEBUG")
]
])
);
$aws_config = array(
'key' => getenv("AWS_ACCESS_KEY_ID"),
'secret' => getenv("AWS_SECRET_ACCESS_KEY"),
'signature_version' => 'v4',
'region' => empty(getenv("AWS_REGION")) ? 'us-east-1' : getenv("AWS_REGION"),
'use_path_style_endpoint' => getenv("AWS_PATH_STYLE") === 'true' ? true : false,
'version' => '2006-03-01',
'scheme' => getenv("AWS_SCHEME") ? getenv("AWS_SCHEME") : "http",
'http' => [
'verify' => getenv("SSL_CERT") === "true" ? getPath('cacert.pem') : false
],
'http_handler' => $handler,
'ua_append' => getenv("SYNO_USER_AGENT")
);
if ("false" === getenv("SDK_RETRY")) {
$aws_config['retries'] = 0;
}
if (!empty(getenv("AWS_URL"))) {
$aws_config["endpoint"] = getenv("AWS_URL");
}
$wrapper = new SynoS3Wrapper($aws_config, $io);
$wrapper->createS3Client($aws_config);
} catch (Exception $e) {
$io->write_exception($e);
return 1;
}
log_debug("\033[33mstart\033[0m");
$io->write_string("start");
while (true) {
try {
$in_json = $io->read_json();
if ($in_json === null) {
$resp = array("success" => false);
if (JSON_ERROR_UTF8 === json_last_error()) {
$resp["error_class"] = "PhpPathContainNonUTF8";
$resp["error_message"] = "path contains non-utf8 chars";
} else if (JSON_ERROR_CTRL_CHAR === json_last_error()) {
$resp["error_class"] = "PhpPathContainCtrlChar";
$resp["error_message"] = "path contains control chars";
}
log_debug("finished with error. (%s)", json_encode($resp));
$io->write_json($resp);
continue;
}
$region = check_need_redirect($wrapper, $aws_config, $in_json);
if (false !== $region) {
$aws_config["region"] = $region;
$aws_config["check_region"] = true;
$wrapper->createS3Client($aws_config);
}
$fn = $in_json["fn"];
unset($in_json["fn"]);
retry_again:
log_debug("\033[33mexec $fn(%s)\033[0m", json_encode($in_json));
$res = call_user_func_array(array($wrapper, $fn), array($in_json));
log_debug("finished");
$io->write_json($res);
} catch (ServerTerminateException $e) {
break;
} catch (RequestException $e) {
$wrapper->createS3Client($aws_config);
$io->write_exception($e);
continue;
} catch (\InvalidArgumentException $e) {
$io->write_exception($e);
continue;
} catch (\Aws\S3\Exception\S3Exception $e) {
if ($e->getStatusCode() === 417 && $aws_config["expect"] !== false) {
$aws_config["expect"] = false;
$wrapper->createS3Client($aws_config);
goto retry_again;
}
if (400 <= $e->getStatusCode() && 499 >= $e->getStatusCode() && can_try_region_from_url($aws_config)) {
if (!array_key_exists("region_from_url_tried", $aws_config)) {
$aws_config["region_from_url_tried"] = false;
$aws_config["region_original"] = $aws_config["region"];
$aws_config["region"] = get_region_from_url($aws_config["endpoint"]);
$wrapper->createS3Client($aws_config);
goto retry_again;
}
// switch back to original region again to response the original exception
if (false === $aws_config["region_from_url_tried"]) {
$aws_config["region_from_url_tried"] = true;
$aws_config["region"] = $aws_config["region_original"];
$wrapper->createS3Client($aws_config);
goto retry_again;
}
}
$io->write_exception($e);
continue;
} catch (Exception $e) {
if (0 == strcmp($e->getMessage(), "read failed")) {
continue;
}
$io->write_exception($e);
continue;
}
}
log_debug("aws_agent terminate");
return 0;
}
if (__name__ == "__main__") {
_main();
}