diff --git a/cron.php b/cron.php index 3b58ee7..fbe7c50 100644 --- a/cron.php +++ b/cron.php @@ -21,21 +21,18 @@ try { $uploadsData = $uploadsCtx->getUploadsData(); // Mark expired as deleted - $expired = $uploadsData->getUploads(expired: true, deleted: false, dmca: false); + $expired = $uploadsData->getUploads(expired: true, deleted: false); foreach($expired as $uploadInfo) $uploadsData->deleteUpload($uploadInfo); // Hard delete soft deleted files - $deleted = $uploadsData->getUploads(deleted: true, dmca: false); + $deleted = $uploadsData->getUploads(deleted: true); foreach($deleted as $uploadInfo) { $uploadsCtx->deleteUploadData($uploadInfo); $uploadsData->nukeUpload($uploadInfo); } - // Ensure local data of DMCA'd files is gone - $deleted = $uploadsData->getUploads(dmca: true); - foreach($deleted as $uploadInfo) - $uploadsCtx->deleteUploadData($uploadInfo); + // new storage format should store by hashes again, ensure blacklisted data is no longer saved } finally { sem_release($semaphore); } diff --git a/database/2023_11_11_015548_new_blacklist_system.php b/database/2023_11_11_015548_new_blacklist_system.php new file mode 100644 index 0000000..ac223a4 --- /dev/null +++ b/database/2023_11_11_015548_new_blacklist_system.php @@ -0,0 +1,18 @@ +execute(' + CREATE TABLE prm_blacklist ( + bl_hash BINARY(32) NOT NULL, + bl_reason ENUM("copyright", "rules", "other") NOT NULL COLLATE "ascii_general_ci", + bl_created TIMESTAMP NOT NULL DEFAULT current_timestamp(), + PRIMARY KEY (bl_hash) + ) ENGINE=InnoDB COLLATE="utf8mb4_bin" + '); + + $conn->execute('ALTER TABLE prm_uploads DROP COLUMN upload_dmca'); + } +} diff --git a/src/Blacklist/BlacklistContext.php b/src/Blacklist/BlacklistContext.php new file mode 100644 index 0000000..1a73bf9 --- /dev/null +++ b/src/Blacklist/BlacklistContext.php @@ -0,0 +1,32 @@ +blacklistData = new BlacklistData($dbConn); + } + + public function getBlacklistData(): BlacklistData { + return $this->blacklistData; + } + + public function createBlacklistEntry(UploadInfo|string $item, string $reason): void { + if($item instanceof UploadInfo) + $item = hex2bin($item->getHashString()); + + $this->blacklistData->createBlacklistEntry($item, $reason); + } + + public function getBlacklistEntry(UploadInfo|string $item): ?BlacklistInfo { + // will this ever be useful? who knows! + if($item instanceof UploadInfo) + $item = hex2bin($item->getHashString()); + + return $this->blacklistData->getBlacklistEntry($item); + } +} diff --git a/src/Blacklist/BlacklistData.php b/src/Blacklist/BlacklistData.php new file mode 100644 index 0000000..619c299 --- /dev/null +++ b/src/Blacklist/BlacklistData.php @@ -0,0 +1,35 @@ +cache = new DbStatementCache($dbConn); + } + + public function getBlacklistEntry(string $hash): ?BlacklistInfo { + $stmt = $this->cache->get('SELECT bl_hash, bl_reason, UNIX_TIMESTAMP(bl_created) FROM prm_blacklist WHERE bl_hash = ?'); + $stmt->addParameter(1, $hash); + $stmt->execute(); + + $result = $stmt->getResult(); + return $result->next() ? new BlacklistInfo($result) : null; + } + + public function createBlacklistEntry(string $hash, string $reason): void { + if(strlen($hash) !== 32) + throw new InvalidArgumentException('$hash must be 32 bytes.'); + if(!in_array($reason, BlacklistInfo::REASONS)) + throw new InvalidArgumentException('$reason is not a valid reason.'); + + $stmt = $this->cache->get('INSERT INTO prm_blacklist (bl_hash, bl_reason) VALUES (?, ?)'); + $stmt->addParameter(1, $hash); + $stmt->addParameter(2, $reason); + $stmt->execute(); + } +} diff --git a/src/Blacklist/BlacklistInfo.php b/src/Blacklist/BlacklistInfo.php new file mode 100644 index 0000000..d90f4b3 --- /dev/null +++ b/src/Blacklist/BlacklistInfo.php @@ -0,0 +1,47 @@ +hash = $result->getString(0); + $this->reason = $result->getString(1); + $this->created = $result->getInteger(2); + } + + public function getHash(): string { + return $this->hash; + } + + public function getReason(): string { + return $this->reason; + } + + public function isCopyrightTakedown(): bool { + return $this->reason === 'copyright'; + } + + public function isRulesViolation(): bool { + return $this->reason === 'rules'; + } + + public function getCreatedTime(): int { + return $this->created; + } + + public function getCreatedAt(): DateTime { + return DateTime::fromUnixTimeSeconds($this->created); + } +} diff --git a/src/EEPROMContext.php b/src/EEPROMContext.php index e69221e..0f6a37d 100644 --- a/src/EEPROMContext.php +++ b/src/EEPROMContext.php @@ -12,6 +12,7 @@ class EEPROMContext { private AuthInfo $authInfo; private Apps\AppsContext $appsCtx; + private Blacklist\BlacklistContext $blacklistCtx; private Uploads\UploadsContext $uploadsCtx; private Users\UsersContext $usersCtx; @@ -22,6 +23,7 @@ class EEPROMContext { $this->authInfo = new AuthInfo; $this->appsCtx = new Apps\AppsContext($dbConn); + $this->blacklistCtx = new Blacklist\BlacklistContext($dbConn); $this->uploadsCtx = new Uploads\UploadsContext($config, $dbConn); $this->usersCtx = new Users\UsersContext($dbConn); } @@ -42,6 +44,10 @@ class EEPROMContext { return $this->appsCtx; } + public function getBlacklistContext(): Blacklist\BlacklistContext { + return $this->blacklistCtx; + } + public function getUploadsContext(): Uploads\UploadsContext { return $this->uploadsCtx; } @@ -67,6 +73,7 @@ class EEPROMContext { $this->authInfo, $this->appsCtx, $this->uploadsCtx, + $this->blacklistCtx, $isApiDomain )); diff --git a/src/LandingRoutes.php b/src/LandingRoutes.php index aa02bec..e528fb6 100644 --- a/src/LandingRoutes.php +++ b/src/LandingRoutes.php @@ -26,7 +26,7 @@ class LandingRoutes extends RouteHandler { $stats->types = 0; $stats->members = 0; - $result = $dbConn->query('SELECT COUNT(upload_id), SUM(upload_size), COUNT(DISTINCT upload_type) FROM prm_uploads WHERE upload_deleted IS NULL AND upload_dmca IS NULL'); + $result = $dbConn->query('SELECT COUNT(upload_id), SUM(upload_size), COUNT(DISTINCT upload_type) FROM prm_uploads WHERE upload_deleted IS NULL'); if($result->next()) { $stats->files = $result->getInteger(0); $stats->size = $result->getInteger(1); diff --git a/src/Uploads/UploadInfo.php b/src/Uploads/UploadInfo.php index 660060c..d6575d3 100644 --- a/src/Uploads/UploadInfo.php +++ b/src/Uploads/UploadInfo.php @@ -16,7 +16,6 @@ class UploadInfo { private ?int $accessed; private ?int $expires; private ?int $deleted; - private ?int $dmca; private int $bump; private string $name; private string $type; @@ -32,11 +31,10 @@ class UploadInfo { $this->accessed = $result->getIntegerOrNull(6); $this->expires = $result->getIntegerOrNull(7); $this->deleted = $result->getIntegerOrNull(8); - $this->dmca = $result->getIntegerOrNull(9); - $this->bump = $result->getInteger(10); - $this->name = $result->getString(11); - $this->type = $result->getString(12); - $this->size = $result->getInteger(13); + $this->bump = $result->getInteger(9); + $this->name = $result->getString(10); + $this->type = $result->getString(11); + $this->size = $result->getInteger(12); } public function getId(): string { @@ -119,18 +117,6 @@ class UploadInfo { return $this->deleted === null ? null : DateTime::fromUnixTimeSeconds($this->deleted); } - public function isCopyrightTakedown(): bool { - return $this->dmca !== null; - } - - public function getCopyrightTakedownTime(): ?int { - return $this->dmca; - } - - public function getCopyrightTakedownAt(): ?DateTime { - return $this->dmca === null ? null : DateTime::fromUnixTimeSeconds($this->dmca); - } - public function getBumpAmount(): int { return $this->bump; } diff --git a/src/Uploads/UploadsData.php b/src/Uploads/UploadsData.php index 8a406b7..321b174 100644 --- a/src/Uploads/UploadsData.php +++ b/src/Uploads/UploadsData.php @@ -17,23 +17,19 @@ class UploadsData { public function getUploads( ?bool $deleted = null, - ?bool $expired = null, - ?bool $dmca = null + ?bool $expired = null ): array { $hasDeleted = $deleted !== null; $hasExpired = $expired !== null; - $hasDMCA = $dmca !== null; $args = 0; - $query = 'SELECT upload_id, user_id, app_id, LOWER(HEX(upload_hash)), INET6_NTOA(upload_ip), UNIX_TIMESTAMP(upload_created), UNIX_TIMESTAMP(upload_accessed), UNIX_TIMESTAMP(upload_expires), UNIX_TIMESTAMP(upload_deleted), UNIX_TIMESTAMP(upload_dmca), upload_bump, upload_name, upload_type, upload_size FROM prm_uploads'; + $query = 'SELECT upload_id, user_id, app_id, LOWER(HEX(upload_hash)), INET6_NTOA(upload_ip), UNIX_TIMESTAMP(upload_created), UNIX_TIMESTAMP(upload_accessed), UNIX_TIMESTAMP(upload_expires), UNIX_TIMESTAMP(upload_deleted), upload_bump, upload_name, upload_type, upload_size FROM prm_uploads'; if($hasDeleted) { ++$args; $query .= sprintf(' WHERE upload_deleted %s NULL', $deleted ? 'IS NOT' : 'IS'); } if($hasExpired) $query .= sprintf(' %s upload_expires %s NOW()', ++$args > 1 ? 'AND' : 'WHERE', $expired ? '<=' : '>'); - if($hasDMCA) - $query .= sprintf(' %s upload_dmca %s NULL', ++$args > 1 ? 'AND' : 'WHERE', $dmca ? 'IS NOT' : 'IS'); $stmt = $this->cache->get($query); $stmt->execute(); @@ -59,7 +55,7 @@ class UploadsData { $hasUserInfo = $userInfo !== null; $args = 0; - $query = 'SELECT upload_id, user_id, app_id, LOWER(HEX(upload_hash)), INET6_NTOA(upload_ip), UNIX_TIMESTAMP(upload_created), UNIX_TIMESTAMP(upload_accessed), UNIX_TIMESTAMP(upload_expires), UNIX_TIMESTAMP(upload_deleted), UNIX_TIMESTAMP(upload_dmca), upload_bump, upload_name, upload_type, upload_size FROM prm_uploads'; + $query = 'SELECT upload_id, user_id, app_id, LOWER(HEX(upload_hash)), INET6_NTOA(upload_ip), UNIX_TIMESTAMP(upload_created), UNIX_TIMESTAMP(upload_accessed), UNIX_TIMESTAMP(upload_expires), UNIX_TIMESTAMP(upload_deleted), upload_bump, upload_name, upload_type, upload_size FROM prm_uploads'; if($hasUploadId) { ++$args; $query .= ' WHERE upload_id = ?'; @@ -158,7 +154,7 @@ class UploadsData { } public function nukeUpload(UploadInfo|string $uploadInfo): void { - $stmt = $this->cache->get('DELETE FROM prm_uploads WHERE upload_id = ? AND upload_dmca IS NULL'); + $stmt = $this->cache->get('DELETE FROM prm_uploads WHERE upload_id = ?'); $stmt->addParameter(1, $uploadInfo instanceof UploadInfo ? $uploadInfo->getId() : $uploadInfo); $stmt->execute(); } diff --git a/src/Uploads/UploadsRoutes.php b/src/Uploads/UploadsRoutes.php index 0e9d83d..562d570 100644 --- a/src/Uploads/UploadsRoutes.php +++ b/src/Uploads/UploadsRoutes.php @@ -7,6 +7,7 @@ use Index\Routing\IRouteHandler; use Index\Routing\Route; use EEPROM\Apps\AppsContext; use EEPROM\Auth\AuthInfo; +use EEPROM\Blacklist\BlacklistContext; use EEPROM\Uploads\UploadsContext; class UploadsRoutes implements IRouteHandler { @@ -14,6 +15,7 @@ class UploadsRoutes implements IRouteHandler { private AuthInfo $authInfo, private AppsContext $appsCtx, private UploadsContext $uploadsCtx, + private BlacklistContext $blacklistCtx, private bool $isApiDomain ) {} @@ -59,9 +61,15 @@ class UploadsRoutes implements IRouteHandler { return 404; } - if($uploadInfo->isCopyrightTakedown()) { - $response->setContent('File is unavailable for copyright reasons.'); - return 451; + $blInfo = $this->blacklistCtx->getBlacklistEntry($uploadInfo); + if($blInfo !== null) { + $response->setContent(match($blInfo->getReason()) { + 'copyright' => 'File is unavailable for copyright reasons.', + 'rules' => 'File was in violation of the rules.', + default => 'File was removed for reasons beyond understanding.', + }); + + return $blInfo->isCopyrightTakedown() ? 451 : 410; } if($uploadInfo->isDeleted() || $uploadInfo->hasExpired()) { @@ -157,19 +165,11 @@ class UploadsRoutes implements IRouteHandler { $uploadsData = $this->uploadsCtx->getUploadsData(); $hash = hash_file('sha256', $localFile); - // this is stupid: dmca status is stored as a file record rather than in a separate table requiring this hack ass garbage - $uploadInfo = $uploadsData->getUpload(appInfo: $appInfo, userInfo: $userInfo, hashString: $hash) - ?? $uploadsData->getUpload(hashString: $hash); - - if($uploadInfo !== null) { - if($uploadInfo->isCopyrightTakedown()) - return 451; - - if($uploadInfo->getUserId() !== $userInfo->getId() - || $uploadInfo->getAppId() !== $appInfo->getId()) - $uploadInfo = null; - } + $blInfo = $this->blacklistCtx->getBlacklistEntry(hex2bin($hash)); + if($blInfo !== null) + return 451; + $uploadInfo = $uploadsData->getUpload(appInfo: $appInfo, userInfo: $userInfo, hashString: $hash); if($uploadInfo === null) { $uploadInfo = $uploadsData->createUpload( $appInfo, $userInfo, $_SERVER['REMOTE_ADDR'], @@ -205,11 +205,6 @@ class UploadsRoutes implements IRouteHandler { return 404; } - if($uploadInfo->isCopyrightTakedown()) { - $response->setContent('File is unavailable for copyright reasons.'); - return 451; - } - if($uploadInfo->isDeleted() || $uploadInfo->hasExpired()) { $response->setContent('File not found.'); return 404;