We first open and analyze the file in ghidra. The main function turns out to be

void FUN_004008a8(void){
int iVar1;
char local_58 [76];
int local_c;

printf("Password: ");
fgets(local_58,0x40,stdin);
local_c = 0;
while( true ) {
iVar1 = FUN_00400699(local_58);
if (iVar1 < local_c) break;
if (local_58[local_c] == '\n') {
local_58[local_c] = '\0';
}
local_c = local_c + 1;
}
FUN_00400874(local_58);
return;
}

Clearly it prints a password and reads input using fgets into chararray local_58. Then FUN_00400699 is called on local_58.

ulong FUN_00400699(long param_1){
int local_10;
uint local_c;

local_c = 0;
local_10 = 0;
while ((local_10 < 0x100 && (*(char *)(param_1 + local_10) != '\n'))) {
local_c = local_c + 1;
local_10 = local_10 + 1;
}
return (ulong)local_c;
}

Clearly, the function returns the length of the input string till ‘\n’ is found. The terminating char ‘\n’ is replaced with ‘\0’. Then FUN_00400874 is called on local_58 in main.

void FUN_00400874(undefined8 param_1){
uint uVar1;

uVar1 = FUN_00400723(param_1);
FUN_00400847((ulong)uVar1);
puts("Correct");
return;
}

The function FUN_00400723 is called on our input string

ulong FUN_00400723(long param_1){
int iVar1;
int iVar2;
int local_10;
uint local_c;

iVar1 = FUN_004006de(param_1);
local_c = iVar1 * 0x1d1dc89d + 0x13ebf05a;
local_10 = 0;
while (local_10 <= iVar1 + -1) {
iVar2 = ((int)((int)*(char *)(param_1 + local_10) * local_c + 0x1eae6490 +
(int)*(char *)(param_1 + local_10)) / 2 ^ (int)*(char *)(param_1 + local_10)) * 3;
if (iVar2 < 0) {
iVar2 = iVar2 + 3;
}
local_c = (iVar2 >> 2 ^ (int)*(char *)(param_1 + local_10)) + iVar1;
if ((int)local_c < 0x10000000) {
local_c = (local_c + 0x13ebf05a) * 6;
}
local_10 = local_10 + 1;
}
return (ulong)local_c;
}

The function performs some sort of hashing on the string and return its val. The function FUN_004006de returns the length of the string till the terminating character ‘\0’.

ulong FUN_004006de(long param_1){
int local_10;
uint local_c;

local_c = 0;
local_10 = 0;
while ((local_10 < 0x100 && (*(char *)(param_1 + local_10) != '\0'))) {
local_c = local_c + 1;
local_10 = local_10 + 1;
}
return (ulong)local_c;
}

Once control is returned to FUN_00400874 FUN_00400847 is called on uvar1 which checks if the value is equal to given hex val. If not, nope is printed and exit.

void FUN_00400847(int param_1){
if (param_1 != -0x4cfb175c) {
puts("Nope");
/* WARNING: Subroutine does not return */
exit(0);
}
return;
}

The hash function can be reversed with

amateur