fetch-xmls

#!/bin/bash

non-empty() {
  for FILE in $(ls -a "$1"); do
    if [ "$FILE" = "." ]; then continue; 
    elif [ "$FILE" = ".." ]; then continue;
    else 
      return 0
    fi
  done
}

XML_DIR=".offdroid-xml"
if [ -d "$XML_DIR" ]; then
  if non-empty "$XML_DIR"; then
    echo "XML directory must be empty; please empty the ./.offdroid-xml directory and try again" 1>&2
    exit 1
  fi
elif [ -e "$XML_DIR" ]; then
  echo ".xml-offdroid must be a directory, but it is not" 1>&2
  exit 2
else
  mkdir "$XML_DIR"
fi

PROXY=""
if [ "$1" = "through" ]; then
  PROXY="$2"
fi

ROOT="http://dl.google.com/android/repository"
HASHLIST="69f54afb9b698723d339154bbe439f93fad84e9a" #list of the hashes found, initially containing the "not found" page's hash

XML_FILEMAP="$XML_DIR/.offdroid-filemap"
XML_URLMAP="$XML_DIR/.offdroid-urlmap"

XML_REPO="repository"
XML_ADDONLIST="addons_list"

NEXT_ENTRY=""
get-and-save() {
  SAVE="$2"
  curl="curl"
  SRC=$1
  if [ -n "$PROXY" ]; then
    curl="curl -x $PROXY"
  fi
  
  echo "GETTING <$SRC>; TO <$SAVE>" 1>&2
  NEXT_ENTRY=""
  HASH=$($curl -s $SRC | tee "$SAVE" | sha1sum) 
  RET=$?

  if [ $RET -eq 0 ]; then
    echo "CHECKING DUPLICATES..." 1>&2
    for H in $HASH; do HASH="$H"; break 1; done
    echo "$HASHLIST" | grep -q -F "$HASH"
    if [ $? -ne 0 ]; then
      echo "FRESH <$HASH>:<$SRC>" 1>&2
      DL=$[$DL+1]
      echo "<$HASH> $SRC" 1>&2
      echo "$HASH $(readlink -f "$SAVE")" >> "$XML_FILEMAP"
      echo "$HASH $SRC" >> "$XML_URLMAP"
      HASHLIST="$HASHLIST $HASH"
      NEXT_ENTRY="$SRC $SAVE"
      echo "get-and-save::NEXT_ENTRY <$NEXT_ENTRY>" 1>&2
    else
      echo "EXISTS: <$HASH>:<$SRC>; HASHLIST: {$HASHLIST}" 1>&2
      rm "$SAVE"
    fi
  else
    echo "error:$RET $SRC"1>&2
  fi

  return 0
}

DL=1
idx=0

XML_REPO_FILE_NAMES=""

MAX_REPO=12
URL="$ROOT/$XML_REPO.xml"
SAVEDIR="$XML_DIR/$DL"
while get-and-save "$URL" "$SAVEDIR"; do
  echo "NEXT_ENTRY <$NEXT_ENTRY>" 1>&2
  if [ -n "$XML_REPO_FILE_NAMES" ]; then
     XML_REPO_FILE_NAMES="$XML_REPO_FILE_NAMES "
  fi

  XML_REPO_FILE_NAMES="$XML_REPO_FILE_NAMES$NEXT_ENTRY"

  if [ $idx -eq $MAX_REPO ]; then
    break 1
  fi

  idx=$[$idx + 1];

  URL="$ROOT/$XML_REPO-$idx.xml"
  SAVEDIR="$XML_DIR/$DL"
done

XML_REPO2="repository2"
idx=0
MAX_REPO2=2
URL="$ROOT/$XML_REPO2.xml"
SAVEDIR="$XML_DIR/$DL"
while get-and-save "$URL" "$SAVEDIR"; do
  echo "NEXT_ENTRY <$NEXT_ENTRY>" 1>&2
  if [ -n "$XML_REPO_FILE_NAMES" ]; then
     XML_REPO_FILE_NAMES="$XML_REPO_FILE_NAMES "
  fi

  XML_REPO_FILE_NAMES="$XML_REPO_FILE_NAMES$NEXT_ENTRY"

  if [ $idx -eq $MAX_REPO2 ]; then
    break 1
  fi

  idx=$[$idx + 1];

  URL="$ROOT/$XML_REPO2-$idx.xml"
  SAVEDIR="$XML_DIR/$DL"
done

idx=0

XML_ADDONLIST_FILE_NAMES=""
MAX_ADDONLIST=5

URL="$ROOT/$XML_ADDONLIST.xml"
SAVEDIR="$XML_DIR/$DL"

while get-and-save "$URL" "$SAVEDIR" ; do
  echo "NEXT_ENTRY <$NEXT_ENTRY>" 1>&2
  if [ -n "$XML_ADDONLIST_FILE_NAMES" ]; then
     XML_ADDONLIST_FILE_NAMES="$XML_ADDONLIST_FILE_NAMES "
  fi

  XML_ADDONLIST_FILE_NAMES="$XML_ADDONLIST_FILE_NAMES$NEXT_ENTRY"
  
  if [ $idx -eq $MAX_ADDONLIST ]; then
    break 1
  fi  

  idx=$[$idx+1]

  URL="$ROOT/$XML_ADDONLIST-$idx.xml"
  SAVEDIR="$XML_DIR/$DL"
done

echo "{REPO: $XML_REPO_FILE_NAMES}" 1>&2

ADDON_URLS=$(node $OFFDROID/get-addons.js addon $XML_ADDONLIST_FILE_NAMES)
for ADDON in $ADDON_URLS; do
  if [ -n "$XML_REPO_FILE_NAMES" ]; then
    XML_REPO_NAMES="$XML_REPO_FILE_NAMES "
  fi
  SAVEDIR="$XML_DIR/$DL"
  get-and-save "$ADDON" "$SAVEDIR"
  XML_REPO_FILE_NAMES="$XML_REPO_FILE_NAMES $ADDON $SAVEDIR"
done