sign method

Implementation

List<int> sign(RSAPrivateKey key, /* String | List<int> | BigInt */ msg,
    {List<int>? salt}) {
  int blockSize = key.blockSize;
  int emBits = key.bitSize - 1;

  List<int> msgBytes;
  if (msg is List<int>) {
    msgBytes = msg;
  } else if (msg is Iterable<int>) {
    msgBytes = msg.toList();
  } else if (msg is String) {
    msgBytes = utf8.encode(msg);
  } else if (msg is BigInt) {
    msgBytes = bigIntToBytes(msg);
  } else {
    throw Exception('invalid message type');
  }

  final mHash = hasher.convert(msgBytes).bytes;
  final hashLength = mHash.length;

  if (emBits < (8 * (hashLength + saltLength) + 9)) {
    throw Exception('emBits too small');
  }

  if (blockSize < hashLength + saltLength + 2) {
    throw Exception('encoding error. blockSize too small');
  }

  if (salt == null) {
    salt =
        List<int>.generate(saltLength, (index) => saltGenerator.nextInt(256));
  } else {
    if (salt.length != saltLength) {
      throw Exception('invalid salt. must be of length $saltLength');
    }
  }

  final mDash = <int>[
    ...List<int>.filled(8, 0),
    ...mHash,
    ...salt,
  ];

  final h = hasher.convert(mDash).bytes;

  final ps = List.filled(blockSize - saltLength - hashLength - 2, 0);

  final db = <int>[
    ...ps,
    0x01,
    ...salt,
  ];

  final dbMask = mgf.encode(blockSize - hashLength - 1, h);

  final maskedDb = ListOps.xor(db, dbMask);

  final em = Uint8List.fromList(<int>[
    ...maskedDb,
    ...h,
    0xbc,
  ]);

  int emDiff = (8 * blockSize) - (emBits);
  if (emDiff < 0) {
    throw Exception();
  } else if (emDiff > 7) {
    throw Exception();
  }
  emDiff = 8 - emDiff;

  em[0] &= (1 << emDiff) - 1;

  return key.engine.signBlock(em);
}