unicode-testscript

#!/bin/bash

## Copyright (C) 2025 - 2025 ENCRYPTED SUPPORT LLC <adrelanos@whonix.org>
## See the file COPYING for copying conditions.

#set -x
set -e
set -o nounset
set -o pipefail
set -o errtrace

if [ "$(id -u)" = 0 ] && [ -z "${GITHUB_WORKSPACE:-}" ]; then
    printf "%s\n" "$0: ERROR: No need to run as root!" >&2
    exit 1
fi

## XXX: Unrelated unit test. Non-ideal location but OK.
source /usr/libexec/helper-scripts/use_sudo.sh
source /usr/libexec/helper-scripts/use_pkexec.sh
source /usr/libexec/helper-scripts/boot-session-detection.bsh

source /usr/libexec/helper-scripts/wc-test.sh

folder_list="Assembly  Bash  C  C#  C++  Go  Java  JavaScript  Python  RegEx  Rust  Solidity"

total_file_counter=0

process_file() {
  local file_name file_content
  file_name="$1"
  test -r "$file_name"
  base_name="$(basename "$file_name")"

  if [ "$base_name" = "README.md" ] || \
    [ "$base_name" = "omnisharp.json" ] || \
    [ "$base_name" = "java.regex" ] || \
    [ "$base_name" = "pcre2.regex" ]; then
    return 0
  fi

  total_file_counter=$(( total_file_counter + 1))
  printf "%s\n" "$file_name" >/dev/null

  ## grep-find-unicode-wrapper is similar like grep.
  ## - Exit code 0: if found.
  ## - Non-zero exit code: if not found.
  if ! grep-find-unicode-wrapper "$file_name" >/dev/null ; then
    printf "%s\n" "$0: ERROR: Failed to find unicode using 'grep-find-unicode-wrapper' in file '$file_name'." >&2
    exit 1
  fi

  ## unicode-show will exit 1 if unicode was found.
  ## So if unicode-show was to exit 0, then it would have failed to find unicode.
  if unicode-show "$file_name" >/dev/null ; then
    ## unicode-show exit code 0.
    printf "%s\n" "$0: ERROR: Failed to find unicode using 'unicode-show' (file check) in file '$file_name'." >&2
    exit 1
  fi

  if cat -- "$file_name" | unicode-show >/dev/null ; then
    ## unicode-show exit code non-zero.
    printf "%s\n" "$0: ERROR: Failed to find unicode using 'unicode-show' (stdin check) in file '$file_name'." >&2
    exit 1
  fi
  ## unicode-show exit code 0.

  ## 'stcatn' should sanitize. Therefore 'unicode-show' should not find any unicode and exit 0.
  ## Check if there is a non-zero exit code in this pipe, because if so, that's a bug.
  if ! stcatn "$file_name" | unicode-show >/dev/null ; then
    ## unicode-show exit code 0.
    printf "%s\n" "$0: ERROR: Failed to sanitize string using 'stcatn' in file '$file_name'." >&2
    exit 1
  fi
  ## unicode-show non-zero exit code.

  if ! cat -- "$file_name" | sanitize-string nolimit >/dev/null ; then
    ## unicode-show exit code 0.
    printf "%s\n" "$0: ERROR: Failed to sanitize string using 'sanitize-string' (stdin check) in file '$file_name'." >&2
    exit 1
  fi

  ## > ./unicode-testscript: line 83: warning: command substitution: ignored null byte in input
  file_content="$(cat -- "$file_name")"

  if ! sanitize-string nolimit "$file_content" >/dev/null ; then
    ## unicode-show exit code 0.
    printf "%s\n" "$0: ERROR: Failed to sanitize string using 'sanitize-string' (command check) in file '$file_name'." >&2
    exit 1
  fi

  return 0
}

if ! test -d "$HOME/trojan-source" ; then
  printf "%s\n" "
$0: ERROR: Folder '$HOME/trojan-source' missing. To get it:

cd ~
git clone git@github.com:nickboucher/trojan-source.git" >&2
fi

## Creates files:
## /tmp/user/1000/suspicious-unicode-test/1.txt
## etc.
/usr/libexec/helper-scripts/write-suspicious-unicode "$HOME/suspicious-unicode-tests-temp-folder/suspicious-unicode-test" >/dev/null

## Create files:
## /tmp/user/1000/suspicious-ascii-test
## etc.
/usr/libexec/helper-scripts/write-suspicious-ascii "$HOME/suspicious-unicode-tests-temp-folder/suspicious-ascii-test" >/dev/null

for folder_name in $folder_list ; do
  while IFS= read -r -d '' file_name; do
    ## Must be outside of process substitution to be able to exit script.
    process_file "$file_name"
  done < <(find "$HOME/trojan-source/$folder_name" -maxdepth 1 -type f -not -iwholename '*.git*' -print0 )
done

while IFS= read -r -d '' file_name; do
  ## Must be outside of process substitution to be able to exit script.
  process_file "$file_name"
done < <(find "$HOME/suspicious-unicode-tests-temp-folder/suspicious-unicode-test" -maxdepth 1 -type f -not -iwholename '*.git*' -print0 )

while IFS= read -r -d '' file_name; do
  ## Must be outside of process substitution to be able to exit script.
  process_file "$file_name"
done < <(find "$HOME/suspicious-unicode-tests-temp-folder/suspicious-ascii-test" -maxdepth 1 -type f -not -iwholename '*.git*' -print0 )

if [ ! "$(printf "%s" "test" | stcatn | wc -l)" = "1" ]; then
  printf "%s\n" "$0: ERROR: 1" >&2
  exit 1
fi

if [ ! "$(printf "%s\n" "test" | stcatn | wc -l)" = "1" ]; then
  printf "%s\n" "$0: ERROR: 2" >&2
  exit 1
fi

if [ ! "$(printf "%s\n\n" "test" | stcatn | wc -l)" = "2" ]; then
  printf "%s\n" "$0: ERROR: 3" >&2
  exit 1
fi

mkdir --parents -- "$HOME/suspicious-unicode-tests-temp-folder/suspicious-whitespace-test"
printf "\n \n " | sponge -- "$HOME/suspicious-unicode-tests-temp-folder/suspicious-whitespace-test/1.txt"

if ! stcatn "$HOME/suspicious-unicode-tests-temp-folder/suspicious-whitespace-test/1.txt" | unicode-show ; then
  printf "%s\n" "$0: ERROR: 4" >&2
  exit 1
fi

if [ ! "$(stcatn "$HOME/suspicious-unicode-tests-temp-folder/suspicious-whitespace-test/1.txt" | wc -l)" = "3" ]; then
  printf "%s\n" "$0: ERROR: 5" >&2
  exit 1
fi

printf "%s\n" "$0: INFO: OK"