Back end

backend/package.json

@@ -21,22 +21,22 @@
   ],
   "main": "index.ts",
   "scripts": {
-    "tsc": "./node_modules/typescript/bin/tsc -p tsconfig.build.json",
+    "tsc": "npx tsc -p tsconfig.build.json",
     "build": "npm run tsc && npm run create-resources",
-    "clean": "rm -rf ./dist/ ./node_modules/ ./package/ ./rust-gbt/",
-    "create-resources": "cp ./src/tasks/price-feeds/mtgox-weekly.json ./dist/tasks && node dist/api/fetch-version.js",
-    "package": "./npm_package.sh",
-    "package-rm-build-deps": "./npm_package_rm_build_deps.sh",
+    "clean": "npx rimraf ./dist/ ./node_modules/ ./package/ ./rust-gbt/",
+    "create-resources": "node scripts/create-resources.js",
+    "package": "node scripts/package.js",
+    "package-rm-build-deps": "node scripts/package-rm-build-deps.js",
     "preinstall": "cd ../rust/gbt && npm run build-release && npm run to-backend",
     "start": "node --max-old-space-size=2048 dist/index.js",
     "start-production": "node --max-old-space-size=16384 dist/index.js",
     "reindex-updated-pools": "npm run start-production --update-pools",
     "reindex-all-blocks": "npm run start-production --update-pools --reindex-blocks",
-    "test": "./node_modules/.bin/jest --coverage",
-    "test:ci": "CI=true ./node_modules/.bin/jest --coverage",
-    "lint": "./node_modules/.bin/eslint . --ext .ts",
-    "lint:fix": "./node_modules/.bin/eslint . --ext .ts --fix",
-    "prettier": "./node_modules/.bin/prettier --write \"src/**/*.{js,ts}\""
+    "test": "npx jest --coverage",
+    "test:ci": "cross-env CI=true npx jest --coverage",
+    "lint": "npx eslint . --ext .ts",
+    "lint:fix": "npx eslint . --ext .ts --fix",
+    "prettier": "npx prettier --write \"src/**/*.{js,ts}\""
   },
   "dependencies": {
     "@babel/core": "^7.25.2",
@@ -46,12 +46,12 @@
     "bip32": "^4.0.0",
     "bitcoinjs-lib": "~6.1.3",
     "crypto-js": "~4.2.0",
-    "express": "~4.21.1",
     "ecpair": "^2.1.0",
+    "express": "~4.21.1",
     "maxmind": "~4.3.11",
     "mysql2": "~3.11.0",
-    "rust-gbt": "file:./rust-gbt",
     "redis": "^4.7.0",
+    "rust-gbt": "file:./rust-gbt",
     "socks-proxy-agent": "~7.0.0",
     "tiny-secp256k1": "^2.2.3",
     "typescript": "~4.9.3",
@@ -67,10 +67,12 @@
     "@types/ws": "~8.5.10",
     "@typescript-eslint/eslint-plugin": "^5.55.0",
     "@typescript-eslint/parser": "^5.55.0",
+    "cross-env": "^7.0.3",
     "eslint": "^8.36.0",
     "eslint-config-prettier": "^8.8.0",
     "jest": "^29.5.0",
     "prettier": "^3.0.0",
+    "rimraf": "^6.0.1",
     "ts-jest": "^29.1.1",
     "ts-node": "^10.9.1"
   }

backend/scripts/create-resources.js

@@ -0,0 +1,21 @@
+const fs = require('fs');
+const path = require('path');
+
+try {
+    // Ensure dist/tasks directory exists
+    const tasksDir = path.join(__dirname, '..', 'dist', 'tasks');
+    fs.mkdirSync(tasksDir, { recursive: true });
+
+    // Copy mtgox-weekly.json
+    const sourceFile = path.join(__dirname, '..', 'src', 'tasks', 'price-feeds', 'mtgox-weekly.json');
+    const targetFile = path.join(tasksDir, 'mtgox-weekly.json');
+    fs.copyFileSync(sourceFile, targetFile);
+
+    // Run fetch-version.js
+    require('../dist/api/fetch-version.js');
+
+    console.log('Resources created successfully');
+} catch (error) {
+    console.error('Error creating resources:', error);
+    process.exit(1);
+}

backend/scripts/final-mysql-setup.sql

@@ -0,0 +1,95 @@
+-- Drop tables in correct order to avoid foreign key constraints
+DROP TABLE IF EXISTS blocks;
+DROP TABLE IF EXISTS pools;
+DROP TABLE IF EXISTS statistics;
+DROP TABLE IF EXISTS elements_pegs;
+DROP TABLE IF EXISTS prices;
+DROP TABLE IF EXISTS state;
+
+-- Create state table first
+CREATE TABLE state (
+    name varchar(25) NOT NULL,
+    number int NULL,
+    string varchar(100) NULL,
+    CONSTRAINT name_unique UNIQUE (name)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci;
+
+-- Insert initial state values
+INSERT INTO state VALUES('schema_version', 83, NULL);
+INSERT INTO state VALUES('last_elements_block', 0, NULL);
+
+-- Create pools table with all required columns
+CREATE TABLE pools (
+    id int NOT NULL AUTO_INCREMENT,
+    unique_id varchar(100) NOT NULL,
+    name varchar(50) NOT NULL,
+    link varchar(255) NOT NULL,
+    addresses json NOT NULL,
+    regexes json NOT NULL,
+    slug varchar(100) NOT NULL,
+    PRIMARY KEY (id),
+    UNIQUE KEY unique_id_idx (unique_id)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci;
+
+-- Insert default pool with all required fields
+INSERT INTO pools (id, unique_id, name, link, addresses, regexes, slug)
+VALUES (
+    -1,
+    'unknown',
+    'Unknown',
+    'https://learnmeabitcoin.com/technical/coinbase-transaction',
+    '[]',
+    '[]',
+    'unknown'
+);
+
+-- Create blocks table with updated schema
+CREATE TABLE blocks (
+    height int unsigned NOT NULL,
+    hash varchar(65) NOT NULL,
+    blockTimestamp timestamp NOT NULL,
+    size int unsigned NOT NULL,
+    weight int unsigned NOT NULL,
+    tx_count smallint unsigned NOT NULL,
+    coinbase_raw text,
+    difficulty double NOT NULL,
+    pool_id int DEFAULT -1,
+    fees double unsigned NOT NULL,
+    reward double unsigned NOT NULL DEFAULT 0,
+    fee_span json NOT NULL,
+    median_fee double unsigned NOT NULL,
+    PRIMARY KEY (height),
+    INDEX pool_idx (pool_id),
+    CONSTRAINT blocks_pool_fk FOREIGN KEY (pool_id) REFERENCES pools(id)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci;
+
+-- Create other required tables
+CREATE TABLE statistics (
+    id int AUTO_INCREMENT PRIMARY KEY,
+    added datetime NOT NULL,
+    unconfirmed_transactions int unsigned NOT NULL,
+    tx_per_second float unsigned NOT NULL,
+    vbytes_per_second int unsigned NOT NULL,
+    mempool_byte_weight int unsigned NOT NULL,
+    fee_data longtext NOT NULL,
+    total_fee double unsigned NOT NULL,
+    INDEX added_idx (added)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci;
+
+-- Create prices table
+CREATE TABLE prices (
+    id SERIAL PRIMARY KEY,
+    time TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    USD DECIMAL(24,8) NOT NULL DEFAULT 0,
+    EUR DECIMAL(24,8) NOT NULL DEFAULT 0,
+    GBP DECIMAL(24,8) NOT NULL DEFAULT 0,
+    CAD DECIMAL(24,8) NOT NULL DEFAULT 0,
+    CHF DECIMAL(24,8) NOT NULL DEFAULT 0,
+    AUD DECIMAL(24,8) NOT NULL DEFAULT 0,
+    JPY DECIMAL(24,8) NOT NULL DEFAULT 0,
+    INDEX time_idx (time)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci;
+
+-- Add initial price data
+INSERT INTO prices (time, USD, EUR, GBP, CAD, CHF, AUD, JPY)
+VALUES (CURRENT_TIMESTAMP, 0, 0, 0, 0, 0, 0, 0);

backend/scripts/migrations.sql

@@ -0,0 +1,87 @@
+-- First drop foreign keys safely
+SET @db_name = DATABASE();
+SET @table_name = 'blocks';
+
+SELECT CONCAT('ALTER TABLE ', @table_name, ' DROP FOREIGN KEY ', CONSTRAINT_NAME, ';')
+INTO @drop_fk_sql
+FROM information_schema.REFERENTIAL_CONSTRAINTS
+WHERE CONSTRAINT_SCHEMA = @db_name
+  AND TABLE_NAME = @table_name
+  AND REFERENCED_TABLE_NAME = 'pools'
+LIMIT 1;
+
+SET @drop_sql = IFNULL(@drop_fk_sql, 'SELECT 1');
+PREPARE stmt FROM @drop_sql;
+EXECUTE stmt;
+DEALLOCATE PREPARE stmt;
+
+-- Drop and recreate tables
+DROP TABLE IF EXISTS blocks;
+DROP TABLE IF EXISTS pools;
+DROP TABLE IF EXISTS statistics;
+DROP TABLE IF EXISTS elements_pegs;
+DROP TABLE IF EXISTS prices;
+DROP TABLE IF EXISTS state;
+
+-- Create state table first
+CREATE TABLE state (
+    name varchar(25) NOT NULL,
+    number int(11) NULL,
+    string varchar(100) NULL,
+    CONSTRAINT name_unique UNIQUE (name)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
+
+-- Insert initial state
+INSERT INTO state VALUES('schema_version', 0, NULL);
+INSERT INTO state VALUES('last_elements_block', 0, NULL);
+
+-- Create pools table
+CREATE TABLE pools (
+    id int(11) NOT NULL AUTO_INCREMENT,
+    name varchar(50) NOT NULL,
+    link varchar(255) NOT NULL,
+    addresses text NOT NULL,
+    regexes text NOT NULL,
+    PRIMARY KEY (id)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
+
+-- Create base blocks table
+CREATE TABLE blocks (
+    height int(11) unsigned NOT NULL,
+    hash varchar(65) NOT NULL,
+    blockTimestamp timestamp NOT NULL,
+    size int(11) unsigned NOT NULL,
+    weight int(11) unsigned NOT NULL,
+    tx_count int(11) unsigned NOT NULL,
+    coinbase_raw text,
+    difficulty bigint(20) unsigned NOT NULL,
+    pool_id int(11) DEFAULT -1,
+    fees double unsigned NOT NULL,
+    fee_span json NOT NULL,
+    median_fee double unsigned NOT NULL,
+    reward double unsigned NOT NULL DEFAULT 0,
+    PRIMARY KEY (height),
+    INDEX (pool_id),
+    FOREIGN KEY (pool_id) REFERENCES pools (id)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
+
+-- Create statistics table
+CREATE TABLE statistics (
+    id int(11) NOT NULL AUTO_INCREMENT,
+    added datetime NOT NULL,
+    unconfirmed_transactions int(11) UNSIGNED NOT NULL,
+    tx_per_second float UNSIGNED NOT NULL,
+    vbytes_per_second int(10) UNSIGNED NOT NULL,
+    mempool_byte_weight int(10) UNSIGNED NOT NULL,
+    fee_data longtext NOT NULL,
+    total_fee double UNSIGNED NOT NULL,
+    PRIMARY KEY (id),
+    INDEX added_idx (added)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
+
+-- Insert default pool
+INSERT INTO pools (id, name, link, addresses, regexes)
+VALUES (-1, 'Unknown', '', '', '');
+
+-- Set schema version to skip migrations
+UPDATE state SET number = 83 WHERE name = 'schema_version';

backend/scripts/mysql8-migrations.sql

@@ -0,0 +1,84 @@
+-- Drop foreign keys safely using information_schema
+SET @table_name = 'blocks';
+SET @constraints = (
+    SELECT GROUP_CONCAT(CONSTRAINT_NAME)
+    FROM information_schema.TABLE_CONSTRAINTS
+    WHERE TABLE_NAME = @table_name
+    AND CONSTRAINT_TYPE = 'FOREIGN KEY'
+    AND TABLE_SCHEMA = DATABASE()
+);
+
+SET @drop_fk_stmt = IF(@constraints IS NOT NULL,
+    CONCAT('ALTER TABLE ', @table_name, ' DROP FOREIGN KEY ', @constraints),
+    'SELECT 1');
+
+PREPARE stmt FROM @drop_fk_stmt;
+EXECUTE stmt;
+DEALLOCATE PREPARE stmt;
+
+-- Drop and recreate tables with proper MySQL 8 syntax
+DROP TABLE IF EXISTS blocks;
+DROP TABLE IF EXISTS pools;
+DROP TABLE IF EXISTS statistics;
+DROP TABLE IF EXISTS elements_pegs;
+DROP TABLE IF EXISTS state;
+
+-- Create state table
+CREATE TABLE state (
+    name varchar(25) NOT NULL,
+    number int NULL,
+    string varchar(100) NULL,
+    CONSTRAINT name_unique UNIQUE (name)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci;
+
+-- Create pools table
+CREATE TABLE pools (
+    id int NOT NULL AUTO_INCREMENT,
+    name varchar(50) NOT NULL,
+    link varchar(255) NOT NULL,
+    addresses text NOT NULL,
+    regexes text NOT NULL,
+    PRIMARY KEY (id)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci;
+
+-- Create blocks table
+CREATE TABLE blocks (
+    height int unsigned NOT NULL DEFAULT 0,
+    hash varchar(65) NOT NULL,
+    blockTimestamp timestamp NOT NULL,
+    size int unsigned NOT NULL DEFAULT 0,
+    weight int unsigned NOT NULL DEFAULT 0,
+    tx_count smallint unsigned NOT NULL DEFAULT 0,
+    coinbase_raw text,
+    difficulty double NOT NULL DEFAULT 0,
+    pool_id int DEFAULT -1,
+    fees double unsigned NOT NULL DEFAULT 0,
+    reward double unsigned NOT NULL DEFAULT 0,
+    fee_span json NOT NULL,
+    median_fee double unsigned NOT NULL DEFAULT 0,
+    INDEX height_idx (height),
+    INDEX pool_idx (pool_id),
+    CONSTRAINT blocks_pool_fk FOREIGN KEY (pool_id) REFERENCES pools(id)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci;
+
+-- Create prices table
+CREATE TABLE prices (
+    id SERIAL PRIMARY KEY,
+    time TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    USD DECIMAL(24,8) NOT NULL DEFAULT 0,
+    EUR DECIMAL(24,8) NOT NULL DEFAULT 0,
+    GBP DECIMAL(24,8) NOT NULL DEFAULT 0,
+    CAD DECIMAL(24,8) NOT NULL DEFAULT 0,
+    CHF DECIMAL(24,8) NOT NULL DEFAULT 0,
+    AUD DECIMAL(24,8) NOT NULL DEFAULT 0,
+    JPY DECIMAL(24,8) NOT NULL DEFAULT 0,
+    INDEX time_index (time)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci;
+
+-- Add initial price data to prevent query errors
+INSERT INTO prices (time, USD, EUR, GBP, CAD, CHF, AUD, JPY)
+VALUES (CURRENT_TIMESTAMP, 0, 0, 0, 0, 0, 0, 0);
+
+-- Initialize state
+INSERT INTO state VALUES('schema_version', 83, NULL);
+INSERT INTO state VALUES('last_elements_block', 0, NULL);