libutil

// util/rc2.c, version 1.0.1
// RC2 source file from libutil
// Copyright (C) 2021, Jakob Wakeling
// MIT Licence

#include "rc2.h"

#include "crypt.h"
#include "endian.h"

#include <assert.h>
#include <stdbool.h>
#include <stdint.h>
#include <stdlib.h>
#include <string.h>

static inline bool isle() { uint16_t e = 0x00FF; return *(uint8_t *)&e; }
static inline bool isbe() { uint16_t e = 0xFF00; return *(uint8_t *)&e; }

static const uint8_t RC2[];

/* Expand a user key to a 128-byte RC2 key. */
void rc2expand(struct rc2 *ctx, const uint8_t *key, int len, int ekl) {
	assert(len >= 1 && len <= 128); assert(ekl <= 1024);
	memcpy(ctx->l, key, len);
	
	int t8 = (ekl + 7) / 8, tm = 0xFF >> (8 * t8 - ekl);
	
	// First key expansion loop
	for (int i = len; i != 128; ++i) {
		ctx->l[i] = RC2[(ctx->l[i - 1] + ctx->l[i - len]) & 0xFF];
	}
	
	// Intermediate step
	ctx->l[128 - t8] = RC2[ctx->l[128 - t8] & tm];
	
	// Second key expansion loop
	for (int i = 128 - t8; i != 0; --i) {
		ctx->l[i - 1] = RC2[ctx->l[i] ^ ctx->l[i + t8 - 1]];
	}
	
	// If CPU is big-endian, then byte swap key
	if (isbe()) for (int i = 0; i != 64; ++i) {
		ctx->k[i] = BSWAP16(ctx->k[i]);
	}
	
	return;
}

/* Encrypt an 8-byte block using RC2 */
void rc2encrypt(struct rc2 *ctx, uint8_t *in, uint8_t *out) {
	uint16_t r0 = LD16LE(in);
	uint16_t r1 = LD16LE(in + 2);
	uint16_t r2 = LD16LE(in + 4);
	uint16_t r3 = LD16LE(in + 6);
	
	for (int i = 0; i != 16; ++i) {
		// Perform mixing round
		r0 += ctx->k[i * 4]     + (r3 & r2) + (~r3 & r1); r0 = ROL16(r0, 1);
		r1 += ctx->k[i * 4 + 1] + (r0 & r3) + (~r0 & r2); r1 = ROL16(r1, 2);
		r2 += ctx->k[i * 4 + 2] + (r1 & r0) + (~r1 & r3); r2 = ROL16(r2, 3);
		r3 += ctx->k[i * 4 + 3] + (r2 & r1) + (~r2 & r0); r3 = ROL16(r3, 5);
		
		// Perform mashing round for rounds 5 and 11
		if (i == 4 || i == 10) {
			r0 += ctx->k[r3 & 63]; r1 += ctx->k[r0 & 63];
			r2 += ctx->k[r1 & 63]; r3 += ctx->k[r2 & 63];
		}
	}
	
	ST16LE(out, r0);
	ST16LE(out + 2, r1);
	ST16LE(out + 4, r2);
	ST16LE(out + 6, r3);
}

/* Decrypt an 8-byte block using RC2 */
void rc2decrypt(struct rc2 *ctx, uint8_t *in, uint8_t *out) {
	uint16_t r0 = LD16LE(in);
	uint16_t r1 = LD16LE(in + 2);
	uint16_t r2 = LD16LE(in + 4);
	uint16_t r3 = LD16LE(in + 6);
	
	for (int i = 15; i >= 0; --i) {
		// Perform r-mixing round
		r3 = ROR16(r3, 5); r3 -= ctx->k[i * 4 + 3] + (r2 & r1) + (~r2 & r0);
		r2 = ROR16(r2, 3); r2 -= ctx->k[i * 4 + 2] + (r1 & r0) + (~r1 & r3);
		r1 = ROR16(r1, 2); r1 -= ctx->k[i * 4 + 1] + (r0 & r3) + (~r0 & r2);
		r0 = ROR16(r0, 1); r0 -= ctx->k[i * 4]     + (r3 & r2) + (~r3 & r1);
		
		// Perform r-mashing round for rounds 5 and 11
		if (i == 5 || i == 11) {
			r3 -= ctx->k[r2 & 63]; r2 -= ctx->k[r1 & 63];
			r1 -= ctx->k[r0 & 63]; r0 -= ctx->k[r3 & 63];
		}
	}
	
	ST16LE(out, r0);
	ST16LE(out + 2, r1);
	ST16LE(out + 4, r2);
	ST16LE(out + 6, r3);
}

static const uint8_t RC2[] = {
	0xD9, 0x78, 0xF9, 0xC4, 0x19, 0xDD, 0xB5, 0xED, 0x28, 0xE9, 0xFD, 0x79,
	0x4A, 0xA0, 0xD8, 0x9D, 0xC6, 0x7E, 0x37, 0x83, 0x2B, 0x76, 0x53, 0x8E,
	0x62, 0x4C, 0x64, 0x88, 0x44, 0x8B, 0xFB, 0xA2, 0x17, 0x9A, 0x59, 0xF5,
	0x87, 0xB3, 0x4F, 0x13, 0x61, 0x45, 0x6D, 0x8D, 0x09, 0x81, 0x7D, 0x32,
	0xBD, 0x8F, 0x40, 0xEB, 0x86, 0xB7, 0x7B, 0x0B, 0xF0, 0x95, 0x21, 0x22,
	0x5C, 0x6B, 0x4E, 0x82, 0x54, 0xD6, 0x65, 0x93, 0xCE, 0x60, 0xB2, 0x1C,
	0x73, 0x56, 0xC0, 0x14, 0xA7, 0x8C, 0xF1, 0xDC, 0x12, 0x75, 0xCA, 0x1F,
	0x3B, 0xBE, 0xE4, 0xD1, 0x42, 0x3D, 0xD4, 0x30, 0xA3, 0x3C, 0xB6, 0x26,
	0x6F, 0xBF, 0x0E, 0xDA, 0x46, 0x69, 0x07, 0x57, 0x27, 0xF2, 0x1D, 0x9B,
	0xBC, 0x94, 0x43, 0x03, 0xF8, 0x11, 0xC7, 0xF6, 0x90, 0xEF, 0x3E, 0xE7,
	0x06, 0xC3, 0xD5, 0x2F, 0xC8, 0x66, 0x1E, 0xD7, 0x08, 0xE8, 0xEA, 0xDE,
	0x80, 0x52, 0xEE, 0xF7, 0x84, 0xAA, 0x72, 0xAC, 0x35, 0x4D, 0x6A, 0x2A,
	0x96, 0x1A, 0xD2, 0x71, 0x5A, 0x15, 0x49, 0x74, 0x4B, 0x9F, 0xD0, 0x5E,
	0x04, 0x18, 0xA4, 0xEC, 0xC2, 0xE0, 0x41, 0x6E, 0x0F, 0x51, 0xCB, 0xCC,
	0x24, 0x91, 0xAF, 0x50, 0xA1, 0xF4, 0x70, 0x39, 0x99, 0x7C, 0x3A, 0x85,
	0x23, 0xB8, 0xB4, 0x7A, 0xFC, 0x02, 0x36, 0x5B, 0x25, 0x55, 0x97, 0x31,
	0x2D, 0x5D, 0xFA, 0x98, 0xE3, 0x8A, 0x92, 0xAE, 0x05, 0xDF, 0x29, 0x10,
	0x67, 0x6C, 0xBA, 0xC9, 0xD3, 0x00, 0xE6, 0xCF, 0xE1, 0x9E, 0xA8, 0x2C,
	0x63, 0x16, 0x01, 0x3F, 0x58, 0xE2, 0x89, 0xA9, 0x0D, 0x38, 0x34, 0x1B,
	0xAB, 0x33, 0xFF, 0xB0, 0xBB, 0x48, 0x0C, 0x5F, 0xB9, 0xB1, 0xCD, 0x2E,
	0xC5, 0xF3, 0xDB, 0x47, 0xE5, 0xA5, 0x9C, 0x77, 0x0A, 0xA6, 0x20, 0x68,
	0xFE, 0x7F, 0xC1, 0xAD
};