new server checking mechanism, complete recode of major functionsv2.4.0

5 files changed, 318 insertions, 724 deletions

diff --git a/lib/MasterServer/UDP/BeaconCatcher.pm b/lib/MasterServer/UDP/BeaconCatcher.pm
index 7c98f57..6058bfa 100755
--- a/lib/MasterServer/UDP/BeaconCatcher.pm
+++ b/lib/MasterServer/UDP/BeaconCatcher.pm
@@ -5,8 +5,7 @@ use warnings;
 use AnyEvent::Handle::UDP;
 use Socket qw(sockaddr_in inet_ntoa);
 use Exporter 'import';
-
-our @EXPORT = qw| beacon_catcher on_beacon_receive|;
+our @EXPORT = qw| beacon_catcher recv_beacon |;
 
 ################################################################################
 ## Receive UDP beacons with \heartbeat\7778\gamename\ut\ format 
@@ -14,64 +13,86 @@ our @EXPORT = qw| beacon_catcher on_beacon_receive|;
 ################################################################################
 sub beacon_catcher {
   my $self = shift;
-  
-  # display that the server is up and listening for beacons
-  $self->log("info", "Listening for UDP beacons on port $self->{beacon_port}.");
-  
   # UDP server
-  my $udp_server; 
-     $udp_server = AnyEvent::Handle::UDP->new(
-  
-    # Bind to this host and use the port specified in the config file
+  my $udp_server; $udp_server = AnyEvent::Handle::UDP->new(
     bind => ['0.0.0.0', $self->{beacon_port}],
-  
-    # when datagrams are received
-    on_recv => sub {$self->on_beacon_receive(@_)},
+    on_recv => sub {$self->recv_beacon(@_)},
   );
-  
-  # allow object to exist beyond this scope. Objects have ambitions too.
+  $self->log("info", "listening for UDP beacons on port $self->{beacon_port}");
   return $udp_server;
 }
 
 ################################################################################
-## Determine the content of the received information and process it.
+# Receive Beacon (Spellchecker suggestion: "Bacon")
+# Check for heartbeats, determine if the server is already in the database
+# or trigger challenge with secure/validate if necessary.
 ################################################################################
-sub on_beacon_receive {
-  # $self, beacon address, handle, packed client address
-  my ($self, $b, $udp, $pa) = @_; 
+sub recv_beacon {
+  # $self, received data, handle, packed client address
+  my ($self, $buffer, $handle, $paddress) = @_;
 
   # unpack ip from packed client address
-  my ($port, $iaddr) = sockaddr_in($pa);
-  my $peer_addr      = inet_ntoa($iaddr);
+  my ($port, $iaddr) = sockaddr_in($paddress);
+  my $beacon_address = inet_ntoa($iaddr);
+
+  # determine and process heartbeat
+  if ($buffer =~ m/\\heartbeat\\/) {
   
-  # assume fraud/crash attempt if response too long
-  if (length $b > 64) {
-    # log
-    $self->log("attack","length exceeded in beacon: $peer_addr:$port sent $b");
+    # process data and get gamename info from the database
+    my $rx = $self->data2hashref($buffer);
     
-    # truncate and try to continue
-    $b = substr $b, 0, 64;
-  }
-
-  # FIXME: note to self: order is important when having combined queries!
-  # TODO:  find a more elegant and long-time solution for this.
+    # some games use heartbeat = 0 because of default ports. Check.
+    if ($rx->{heartbeat} == 0 && $rx->{gamename}) {
+    
+      # overwrite the heartbeat port with a known default port, or zero
+      $rx->{heartbeat} = $self->get_game_props(gamename => $rx->{gamename})->[0]->{default_qport} || 0;
+      
+      # if no default port is listed, log and return. !! can spam the logs !!
+      if ($rx->{heartbeat} == 0) {
+        $self->log("invalid", "$beacon_address has no default heartbeat port listed");
+        return;
+      }
+    }
 
-  # if this is a secure response, verify the response
-  $self->process_udp_validate($b, $peer_addr, $port, undef) 
-    if ($b =~ m/\\validate\\/);
+    # update the timestamp in the database if the server already exists
+    my $upd = $self->update_server(
+      ip        => $beacon_address, 
+      port      => $rx->{heartbeat}, 
+      direct    => 1,
+    );
+    
+    # did the update succeed?
+    if ($upd > 0) {
+      # then we're done here. log and return.
+      $self->log("beacon", "heartbeat from $beacon_address, $rx->{heartbeat}". 
+        ($rx->{gamename} ? (" for $rx->{gamename}") : "") );
+    } 
+    # if no update occurred, query server
+    else {
+      # assign BeaconChecker to query the server for secure challenge and status
+      $self->query_udp_server(
+        ip    => $beacon_address,
+        port  => $rx->{heartbeat},
+        need_validate => 1,
+        direct_uplink => 1,
+      );
+    }
+    return;
+  }
 
-  # if a heartbeat format was detected...
-  $self->process_udp_beacon($udp, $pa, $b, $peer_addr, $port) 
-    if ($b =~ m/\\heartbeat\\/ && $b =~ m/\\gamename\\/);
+  # other masterservers check if we're still alive, respond with complient data
+  if ($buffer =~ m/\\(secure|basic|rules|info|players|status)\\/i) {
+    $self->handle_status_query($handle, $paddress, $buffer);
+    $self->log("uplink", "responding to $beacon_address, $port (sent $buffer)");
+    return;
+  }
   
-  # if other masterservers check if we're still alive
-  $self->handle_status_query($udp, $pa, $b, $peer_addr) 
-    if ($b =~ m/\\secure\\/ || 
-        $b =~ m/\\basic\\/  || 
-        $b =~ m/\\info\\/   || 
-        $b =~ m/\\rules\\/  ||
-        $b =~ m/\\players\\/|| 
-        $b =~ m/\\status\\/);
+  # Util::UDPBrowser (optional)
+  if ($buffer =~ m/^\\echo\\request/i) {
+    $self->udpbrowser_host($handle, $paddress, $buffer);
+    return;
+  }
+
 }
 
 1;
diff --git a/lib/MasterServer/UDP/BeaconChecker.pm b/lib/MasterServer/UDP/BeaconChecker.pm
index 73220cf..9c95455 100755
--- a/lib/MasterServer/UDP/BeaconChecker.pm
+++ b/lib/MasterServer/UDP/BeaconChecker.pm
@@ -4,80 +4,52 @@ use strict;
 use warnings;
 use AnyEvent::Handle::UDP;
 use Exporter 'import';
-
 our @EXPORT = qw| query_udp_server |;
 
 ################################################################################
 ## Get the server status from any server over UDP and store the received 
 ## information in the database. $secure determines the type of query: 
 ## secure/pending or information.
+## options: ip, port, need_validate, direct_uplink
 ################################################################################
 sub query_udp_server {
-  my ($self, $id, $ip, $port, $secure, $message_type) = @_;
-  my $buf = "";
-  
-  # debug logging
-  # $self->log("debug", "Query server $id ($ip:$port)");
+  my ($self, %o) = @_;
+  my $buffer = "";
+
+  # if a secure/validate challenge is still required, generate secure string
+  my $secure = $self->secure_string if $o{need_validate};
   
   # connect with UDP server
   my $udp_client; $udp_client = AnyEvent::Handle::UDP->new(
-    connect   => [$ip, $port],
-    timeout   => $self->{timeout_time},
-    on_timeout => sub {$udp_client->destroy();}, # do not report timeouts
-    on_error   => sub {$udp_client->destroy();}, # or errors
+    connect    => [$o{ip}, $o{port}],
+    timeout    => $self->{timeout_time},
+    on_timeout => sub {$udp_client->destroy;},
+    on_error   => sub {$udp_client->destroy;},
     on_recv    => sub {
-
-      # add packet to buffer
-      $buf .= $_[0];
-
-      # FIXME: note to self: order is important when having combined queries!
-      # TODO:  find a more elegant and long-time solution for this.
+      # add received data to buffer
+      $buffer .= $_[0];
       
-      # message type 1: \basic\\secure\wookie
-      # if validate, assume that we sent a \basic\secure request.
-      if ($buf =~ m/\\validate\\/){
-        $self->process_udp_validate($buf, $ip, undef, $port);
+      # buffer completed receiving all relevant information?
+      if ($buffer =~ m/\\final\\/) {
+
+        # try to process datagram
+        $self->process_datagram(
+          ip      => $o{ip},
+          port    => $o{port},
+          rxbuf   => $buffer,
+          secure  => $secure,
+          direct  => $o{direct_uplink},
+        );
       }
       
-      # message type 0: \basic\\info\
-      # if gamename, ver, hostname and hostport are available, but NOT the value 
-      # "listenserver", it would have been \basic\info
-      if ($buf =~ m/\\gamename\\/ && 
-          $buf =~ m/\\hostname\\/ && 
-          $buf =~ m/\\hostport\\/ &&
-          $buf !~ m/\\listenserver\\/ ) {
-        $self->process_query_response($buf, $ip, $port);
-      }
-      
-      # message type 2: \status\
-      # contains same info as \basic\\info, but also "listenserver". Only for UT.
-      if ($buf =~ m/\\gamename\\ut/ && 
-          $buf =~ m/\\hostname\\/   && 
-          $buf =~ m/\\hostport\\/   &&
-          $buf =~ m/\\listenserver\\/ ) {
-        $self->process_status_response($buf, $ip, $port);
-      }
-
       # else partial information received. wait for more.
       # else { }
     },
   );
 
-  #
-  # Send secure message or status, depending on provided variables
-  # Message types can be 
-  #   0: \basic\\info\
-  #   1: \basic\\secure\wookie
-  #   2: \status\
-  #
-  
-  # determine the message
-  my $message = "\\basic\\\\info\\"; # default 0
-     $message = "\\basic\\\\secure\\$secure" if ($secure ne "" && $self->{require_secure_beacons} > 0); # message_type 1
-     $message = "\\status\\" if ($message_type == 2);
-
-  # send selected message
-  $udp_client->push_send($message);
+  # determine the requests and send message
+  $udp_client->push_send("\\secure\\$secure") if $o{need_validate};
+  $udp_client->push_send("\\status\\");
 }
 
 1;
diff --git a/lib/MasterServer/UDP/DatagramProcessor.pm b/lib/MasterServer/UDP/DatagramProcessor.pm
index 006871d..5587875 100755
--- a/lib/MasterServer/UDP/DatagramProcessor.pm
+++ b/lib/MasterServer/UDP/DatagramProcessor.pm
@@ -2,393 +2,166 @@ package MasterServer::UDP::DatagramProcessor;
 
 use strict;
 use warnings;
-use Encode;
-use AnyEvent::Handle::UDP;
 use Exporter 'import';
+our @EXPORT = qw| process_datagram |;
 
-our @EXPORT = qw| process_udp_beacon 
-                  process_udp_validate 
-                  process_query_response 
-                  process_status_response
-                  process_ucc_applet_query |;
-
-################################################################################
-## Process datagrams from beacons that have \heartbeat\ and \gamename\ keys
-## in the stringbuffer. If necessary, authenticate first with the secure/val
-## challenge.
-################################################################################
-sub process_udp_beacon {
-  # $self, handle, packed address, udp data, peer ip address, $port
-  my ($self, $udp, $pa, $buf, $peer_addr, $port) = @_; 
-  
-  # received heartbeat in $buf: \heartbeat\7778\gamename\ut\ 
-  my %r;
-  my $raw = $buf; # raw buffer for logging if necessary
-  $buf = encode('UTF-8', $buf);
-  $buf =~ s/\\([^\\]+)\\([^\\]+)/$r{$1}=$2/eg;
-  
-
-  # check whether the beacon has a gamename
-  if (defined $r{gamename}) {
-    # log the beacon
-    $self->log("beacon", "$peer_addr:$r{heartbeat} for $r{gamename}");
-    
-    # check if game is actually supported in our db
-    my $game_props = $self->get_game_props(gamename => $r{gamename})->[0];
-    
-    # if no entry exists, report error.
-    if (defined $game_props) {
-      
-      # validate heartbeat data
-      my $heartbeat = ($r{heartbeat} || ($game_props->{default_qport} || 0));
-      
-      #
-      # verify valid server address (ip+port)
-      if ($self->valid_address($peer_addr,$heartbeat)) {
-      
-        # check if the entry already was not added within the last 5 seconds, throttle otherwise
-        my $throttle = $self->get_pending(
-          ip        => $peer_addr, 
-          heartbeat => $heartbeat, 
-          gamename  => $r{gamename},
-          after     => 5,
-          sort      => "added",
-          limit     => 1
-        )->[0];
-        return if (defined $throttle);
-        
-        # generate a new secure string
-        my $secure = $self->secure_string();
-        
-        # update beacon in serverlist if it already exists, otherwise update
-        # or add to pending with new secure string.
-        my $auth = $self->add_server_new(ip         => $peer_addr, 
-                                         beaconport => $port, 
-                                         heartbeat  => $heartbeat, 
-                                         gamename   => $r{gamename}, 
-                                         secure     => $secure,
-                                         direct     => 1,
-                                         updated    => time,
-                                         beacon     => time);
-
-        # send secure string back
-        if ($auth > 0) {
-          
-          # verify that this is a legitimate client by sending the "secure" query
-          $udp->push_send("\\secure\\$secure\\final\\", $pa);
-            
-          # log this as a new beacon (debug)
-          #$self->log("secure", "challenged new beacon $peer_addr:$port with $secure.");
-          }
-      }
-      
-      # invalid ip+port combination, like \heartbeat\0\ or local IP
-      else {
-        # Log that beacon had incorrect information, such as port 0 or so. Spams log!
-        $self->log("invalid","$peer_addr had bad information --> $raw");
-      }
-    
-    }
-    # unknown game
-    else {
-      $self->log("support","$peer_addr tries to identify as unknown game \"$r{gamename}\".");
-    }
-    
-  }
-  
-  # gamename not valid or recognized, display raw buffer in case data could not 
-  # be extrapolated from the heartbeat
-  else {
-    # log
-    $self->log("support", "received unknown beacon from $peer_addr --> '$raw'");
-  }
-}
-
-################################################################################
-## Process the received validate query and determine whether the server is 
-## allowed in our database. Either provide heartbeat OR port, not both.
 ################################################################################
-sub process_udp_validate {
-  # $self, udp data, ip, port
-  my ($self, $buf, $peer_addr, $port, $heartbeat) = @_;
-  
-  # debug spamming
-  # $self->log("udp", "Received response from $peer_addr:$heartbeat, sent |$buf|");
-  
-  # received heartbeat in $b:    \validate\string\queryid\99.9\ 
-  my %r;
-  $buf = encode('UTF-8', $buf);
-  $buf =~ s/\\([^\\]+)\\([^\\]+)/$r{$1}=$2/eg;
-  
-  # get our existing knowledge about this server from the database 
-  my $pending = $self->get_pending(
-        ip          => $peer_addr, 
-        limit       => 1,
-        ($heartbeat ? (heartbeat   =>  $heartbeat) : () ),
-        ($port      ? (beaconport  =>  $port)      : () ),
-  )->[0];
-  
-  # if indeed in the pending list, check; -- if this entry is not (longer) in the list, it
-  # was either removed by the BeaconChecker or cleaned out in maintenance (after X hours).
-  if (defined $pending) {
-    
-    # determine if it uses any enctype
-    my $enc = (defined $r{enctype}) ? $r{enctype} : 0;
-    
-    # database may not contain the correct gamename (ucc applet, incomplete beacon, other game)
-    $pending->{gamename} = $r{gamename} if (defined $r{gamename});
-
-    # verify challenge
-    my $val = $self->compare_challenge(
-                gamename => lc $pending->{gamename},
-                secure   => $pending->{secure},
-                enctype  => $r{enctype},
-                validate => $r{validate},
-                ignore   => $self->{ignore_beacon_key},
-              );
+## Process datagrams after querying a server.
+## %o contains ip, port, recv buffer, secure string
+################################################################################                  
+sub process_datagram {
+  my ($self, %o) = @_;
+  my $rx = $self->data2hashref($o{rxbuf});
+
+  # can not proceed if validate was provided, but not gamename
+  return 0 if ( $rx->{validate} && not($rx->{gamename}) );
+  # do not process data if no hostport was provided.
+  return 0 unless $rx->{hostport};
+
+  # truncate excessively long fields like hostname
+  $rx->{hostname} = substr $rx->{hostname}, 0, 199 if (length $rx->{hostname} >= 199);
+  
+  # try updating serverlist info based on ip/hostport
+  my $update = $self->update_server(
+    ip        => $o{ip},
+    hostport  => $rx->{hostport},
+    direct    => $o{direct},
+    %{$rx},
+  );
+
+  # if not found, insert it in the table, after verification
+  if ($update == 0) {
+    # can not proceed if gamename was provided, but not validate
+    return 0 if ( not($rx->{validate}) && $rx->{gamename} );
+  
+    # does the recv buffer contain a validation segment?
+    my $auth = $self->auth_server(
+      gamename  => lc $rx->{gamename},
+      secure    => $o{secure},
+      enctype   => $rx->{enctype},
+      validate  => $rx->{validate},
+    ) if ($rx->{validate} && $rx->{gamename});
+
+    # if authenticated, or known to be incapable of authenticating (tribesv)
+    if ($auth || $self->{secure_unsupported} =~ m/$rx->{gamename}/i ) {
+      # add to the database in three steps. First, insert basic data.
+      $self->insert_server(
+        ip        => $o{ip},
+        port      => $o{port},
+        hostport  => $rx->{hostport},
+      );
+      # second, update the entry with all available information
+      $self->update_server(
+        ip        => $o{ip},
+        hostport  => $rx->{hostport},
+        direct    => $o{direct},
+        %{$rx},
+      );
+      # third, insert an entry for extended server information
+      $self->insert_extended(
+        ip        => $o{ip}, 
+        hostport  => $rx->{hostport}
+      );
+      # log new beacon
+      $self->log("add", "new server $o{ip}, $rx->{hostport}". 
+        ($rx->{gamename} ? (" for $rx->{gamename}") : "") );
 
-    # if validated, add server to database
-    if ($val > 0 || $self->{require_secure_beacons} == 0) {
-      
-      # select server from serverlist -- should not exist yet.
-      my $srv = $self->get_server(ip => $pending->{ip}, port => $pending->{heartbeat})->[0];
-      
-      # was found, then update gamename and remove from pending
-      if (defined $srv) {
-        my $sa = $self->update_server_list(
-           ip       => $pending->{ip}, 
-           port     => $pending->{heartbeat},
-           gamename => $pending->{gamename}
-        );
-        # remove the entry from pending if successfully added
-        $self->remove_pending($pending->{id}) if ( $sa >= 0);
-      }
-      # was not found in serverlist, insert clean and remove from pending
-      else {
-        my $sa = $self->add_server_list(
-           ip       => $pending->{ip}, 
-           port     => $pending->{heartbeat},
-           gamename => $pending->{gamename}
-        );
-        # remove the entry from pending if successfully added
-        $self->remove_pending($pending->{id}) if ( $sa > 0);
-      }
+      # addresses are often added through pending list. delete if successful
+      $self->remove_pending(ip => $o{ip}, port => $o{port});
     }
     else {
-      # else failed validation
-      # calculate expected result for log
-      
-      my $validate_string = "";
-      if ($pending->{gamename} && $pending->{secure}) {
-        $validate_string = $self->validate_string(
-          gamename => $pending->{gamename}, 
-          secure => $pending->{secure}
-        );
-      }
-      $self->log("secure","$pending->{id} for ". 
-                          ($pending->{gamename} || "empty_p_gamename")
-        ." sent: '".      ($pending->{secure}   || "empty_p_secure")
-        ."', expected '". ($validate_string     || "empty_v_string")
-        ."', got '".      ($r{validate}         || "empty_r_validate")
+      # log: failed secure test
+      my $val_str = $self->validate_string(
+        gamename  => lc $rx->{gamename},
+        secure    => $o{secure},
+        enctype   => $rx->{enctype},
+        validate  => $rx->{validate},
+      );
+      $self->log("secure","$o{ip}, $o{port} failed validation for ". 
+                          ($rx->{gamename} || "empty_gamename")
+        ."; sent: '".     ($o{secure}      || "empty_secure")
+        ."', expected '". ($val_str        || "empty_v_string")
+        ."', got '".      ($rx->{validate} || "empty_r_validate")
+        ."' with cipher '". ($self->get_game_props(gamename => $rx->{gamename})->[0]->{cipher} || "empty_cipher")
         ."'"
       );
+
+      # remove addresses anyway to prevent error spamming in log
+      $self->remove_pending(ip => $o{ip}, port => $o{port});
+      return 0;
     }
   }
-  # if no entry found in pending list
-  else {
-    # not found
-    $self->log("error","server not found in pending for ".
-      ($peer_addr || "ip") .":".
-      ($heartbeat || "0")  .",".
-      ($port      || "0")  ." !");
-  }
-}
-
-################################################################################
-## Process query data that was obtained with \basic\ and/or \info\ from the
-## beacon checker module.
-## FIXME: error checking and data processing. ($_ || "default") instead.
-################################################################################
-sub process_query_response {
-  # $self, udp data, ip, port
-  my ($self, $buf, $ip, $port) = @_;
-
-  # process datastream
-  my %s = ();
-  $buf = encode('UTF-8', $buf);
-  $buf =~ s/\\([^\\]+)\\([^\\]+)/$s{$1}=$2/eg;
   
+  # select server id for faster/easier referencing
+  my $sid = $self->get_server(
+    ip        => $o{ip}, 
+    hostport  => $rx->{hostport}, 
+    limit     => 1
+  )->[0]->{id} || 0;
   
-  
-  # check whether the gamename is supported in our db
-  if (exists $s{gamename} && $self->get_game_props(gamename => $s{gamename})) {
-  
-    # parse variables
-    my %nfo = ();
-    $nfo{gamever}   = exists $s{gamever}  ? $s{gamever}   : "";
-    $nfo{hostname}  = exists $s{hostname} ? $s{hostname}  : "$ip:$port";
-    $nfo{hostport}  = exists $s{hostport} ? $s{hostport}  : 0;
-    
-    # some mor0ns have hostnames longer than 200 characters
-    $nfo{hostname} = substr $nfo{hostname}, 0, 199 if (length $nfo{hostname} >= 199);
-    
-    # log results (debug)
-    # $self->log("hostname", "$ip:$port is now known as $nfo{hostname}");
-    
-    # add or update in serverlist (assuming validation is complete)
-    my $result = $self->update_server_list(
-        ip          => $ip, 
-        port        => $port, 
-        gamename    => $s{gamename},
-        %nfo);
-
-    # if address is in pending list, remove it
-    my $pen = $self->get_pending(ip => $ip, heartbeat => $port)->[0];
-    $self->remove_pending($pen->{id}) if $pen;
-    
-    # log potential error
-    $self->log("support", "no entries were updated for $ip:$port ($s{gamename}), but it was still removed from pending!") if ($result == 0 && $pen);
-  }
-}
+  # server not found in db. strange. manually deleted? ignore and return.
+  return 0 unless $sid;
 
-################################################################################
-## Process status data that was obtained with \status\ from the
-## UT serverstats checker module.
-################################################################################
-sub process_status_response {
-  # $self, udp data, ip, port
-  my ($self, $buf, $ip, $port) = @_;
-
-  # process datastream
-  my %s;
-  $buf = encode('UTF-8', $buf);
-  $buf =~ s/\\([^\\]+)\\([^\\]+)/$s{$1}=$2/eg;
+  # update extended information with the unified/new info columns
+  my ($uei, $upi) = unify_information($sid,$rx);
+  my $u = $self->update_extended(sid => $sid, %{$uei});
   
-  # check whether this server is in our database
-  my $serverlist_id = $self->get_server(ip => $ip, port => $port)->[0];
+  # update player information (first delete, then add new)
+  $self->delete_players($sid);
+  for my $pl (@{$upi}) {$self->insert_players(@{$pl});}
   
-  # only allow servers that were approved/past pending
-  if (defined $serverlist_id) {
-
-    #
-    # pre-process variables before putting them in the db
-    #
-    
-    # gamename should in all cases be "ut" (we only allow this for UT games at the moment!)
-    return if (!defined $s{gamename} || $s{gamename} ne "ut");
-    
-    # some people trying to sneak their Unreal servers into the UT serverlist
-    return if (!defined $s{gamever} || $s{gamever} eq "227i");
-    
-    # some sanity checks for the presentation
-    $s{hostname} = substr $s{hostname}, 0, 199 if ($s{hostname} && length $s{hostname} >= 199);
-    $s{mapname}  = substr $s{mapname},  0,  99 if ($s{mapname}  && length $s{mapname}  >= 99);
-    $s{maptitle} = substr $s{maptitle}, 0,  99 if ($s{maptitle} && length $s{maptitle} >= 99);
-
-    #
-    # Store info in database
-    #
-
-    # check if the ID already exists in the database
-    my $utserver_id = $self->get_utserver(id => $serverlist_id->{id})->[0];
-    
-    # add and/or update
-    $self->add_utserver($ip, $port) if (not defined $utserver_id);
-    $self->update_utserver($serverlist_id->{id}, %s);
-    
-    #
-    # Player info
-    #
-    
-    # delete all players for this server.
-    $self->delete_utplayers($serverlist_id->{id});
-    
-    # iterate through all player IDs and add them to the database
-    for (my $i = 0; exists $s{"player_$i"}; $i++) {
-      
-      # shorten name (some people might be overcompensating their names)
-      $s{"player_$i"} = substr $s{"player_$i"}, 0, 39 if (length $s{"player_$i"} > 39);
-      
-      my %player = ();
-      $player{player}   = exists $s{"player_$i"}   ?     $s{"player_$i"}     : "Player";
-      $player{team}     = exists $s{"team_$i"}     ?     $s{"team_$i"}     : 255;
-      $player{team}     = ($player{team} =~ m/^[0-3]/ ) ? int($player{team}) : 255;
-      $player{frags}    = exists $s{"frags_$i"}    ? int($s{"frags_$i"})   : 0;
-      $player{mesh}     = exists $s{"mesh_$i"}     ?     $s{"mesh_$i"}     : "";
-      $player{skin}     = exists $s{"skin_$i"}     ?     $s{"skin_$i"}     : "";
-      $player{face}     = exists $s{"face_$i"}     ?     $s{"face_$i"}     : "";
-      $player{ping}     = exists $s{"ping_$i"}     ? int($s{"ping_$i"})    : 0;
-      $player{ngsecret} = exists $s{"ngsecret_$i"} ?     $s{"ngsecret_$i"} : ""; # contains bot info
-      
-      # write to db
-      $self->insert_utplayer($serverlist_id->{id}, %player);
-    }
-
-    # log results (debug)
-    #$self->log("utserver", 
-    #  "$serverlist_id->{id}, $ip:$port,\t".
-    #  ($s{numplayers} || "0") ."/".
-    #  ($s{maxplayers} || "0") ."players, ".
-    #  ($s{mapname}    || "mapname")  .",".
-    #  ($s{hostname}   || "hostname")
-    #);
-  }
-}
-
-################################################################################
-## Process the list of addresses that was received after querying the UCC applet
-## and store them in the pending list.
-################################################################################
-sub process_ucc_applet_query {
-  my ($self, $buf, $ms) = @_;
-  $buf = encode('UTF-8', $buf);
+  # return true when all done
+  return 1 if int($u || 0);
   
-  # counter
-  my $c = 0;
+  # update possibly failed because we migrated from an older serverlist.
+  $self->log("warning", "no extended information for $o{ip}, $rx->{hostport} to update");
   
-  # database types such as SQLite are slow, therefore use transactions.
-  $self->{dbh}->begin_work;
+  # insert extended table entry again
+  $self->insert_extended(
+    ip        => $o{ip}, 
+    hostport  => $rx->{hostport}
+  );
   
-  # parse $buf into an array of [ip, port]
-  foreach my $l (split(/\\/, $buf)) {
+  # and try to update it again (players were already added independently)
+  $u = $self->update_extended(sid => $sid, %{$uei});
   
-    # search for \ip\255.255.255.255:7778\, contains ':'
-    if ($l =~ /:/) {
-      my ($a,$p) = $l =~ /(.*):(.*)/;
-      
-      # check if address entry is valid
-      if ($self->valid_address($a,$p)) {
-        # count number of valid addresses
-        $c++;
-        
-        # print address (debug)
-        # $self->log("add", "applet query added $ms->{gamename}\t$a\t$p");
-        
-        # add server
-        $self->add_server_new(ip         => $a,
-                              beaconport => $p,
-                              heartbeat  => $p, 
-                              gamename   => $ms->{gamename}, 
-                              secure     => $self->secure_string(),
-                              updated    => time);
-      }
-      # invalid address, log
-      else {$self->log("error", "invalid address found at master applet $ms->{ip}, $l!");}
-    }
-  }
+  # return true when all done
+  return 1 if int($u || 0);
   
-  # end transaction, commit        
-  $self->{dbh}->commit;
+  # now we're toast
+  $self->log("error", "failed to insert $o{ip}, $rx->{hostport} extended information twice");
+  return 0;
+}
 
-  # update time if successful applet query
-  $self->update_master_applet(
-    ip        => $ms->{ip},
-    port      => $ms->{port},
-    gamename  => $ms->{gamename}, 
-  ) if ($c > 0);
-  
-  # print findings
-  $self->log("applet-rx","found $c addresses at $ms->{ip} for $ms->{gamename}.");
+################################################################################
+## Process data into readable player stat columns
+## server id, received data buffer hash
+## returns unified extended info, unified player info
+################################################################################     
+sub unify_information {
+  my ($sid, $rx) = @_;
+  my %uei; # unified extended info
+  my @upi; # unified  player  info
+  
+  # FIXME unify with player playername name
+
+  # first process all available player entries
+  for (my $i = 0; exists $rx->{"player_$i"}; $i++) {
+    # add player info to UPI and remove from hash
+    my @player;
+    push @player, $sid;
+    push @player, delete $rx->{"player_$i"} || "Derp";
+    push @player, delete $rx->{"team_$i"};
+    push @player, int (delete $rx->{"frags_$i"} || 0);
+    push @player, delete $rx->{"mesh_$i"};
+    push @player, delete $rx->{"skin_$i"};
+    push @player, delete $rx->{"face_$i"};
+    push @player, int (delete $rx->{"ping_$i"} || 0);
+    push @player, delete $rx->{"ngsecret_$i"};
+    push @upi, \@player;
+  }
+  # return remaining values, player array
+  return ($rx, \@upi);
 }
 
 1;
diff --git a/lib/MasterServer/UDP/UDPTicker.pm b/lib/MasterServer/UDP/UDPTicker.pm
index 6b3a681..0566449 100755
--- a/lib/MasterServer/UDP/UDPTicker.pm
+++ b/lib/MasterServer/UDP/UDPTicker.pm
@@ -4,253 +4,93 @@ use strict;
 use warnings;
 use AnyEvent::Handle::UDP;
 use Exporter 'import';
-use Data::Dumper 'Dumper';
-
 our @EXPORT = qw| udp_ticker |;
 
 ################################################################################
-##   When addresses are stored in the 'pending' list, they are supposed to be
-##   queried immediately with the secure/validate challenge to testify that
-##   the server is genuine and alive.
-##
-##   Some servers do not support the secure-challenge on the Uplink port. These
-##   servers are verified with a secure-challenge on their heartbeat ports, 
-##   which are designed to respond to secure queries, as well as status queries.
-##
-##   Addresses collected by other scripts, whether from the UCC applet or manual
-##   input via the website, are added to the pending list. It is more 
-##   important to verify pending beacons and new server addresses, than to 
-##   update the status of existing addresses. Therefore, pending addresses are
-##   prioritized.
+##   When addresses are provided from secondary sources (master applets, 
+##   synchronization or manual addition, they are queried by this udp_ticker.
+##   When they validate (which also implies correct router settings) they are
+##   added to the masterserver list.
 ##
-##   Another function required for 333networks is the "server info" part of the
-##   site. UT servers are queried and stored in the database. This is the lowest
-##   priority for the masterserver and is therefore performed last.
+##   Some servers do not support the secure-challenge or do not respond to 
+##   queries directly. By retrieving the server information we are able to 
+##   make exceptions on a case to case basis.
 ##
+##   Other than previous MS-Perl versions, unresponsive servers are no longer
+##   checked. When servers become fail to report in after 2 hours, they remain 
+##   are considered offline and will remain archived. This server can become
+##   active again by uplinking to one of the affiliated masterservers.
 ################################################################################
 sub udp_ticker {
   my $self = shift;
 
-  # inform that we are running 
-  $self->log("info", "UDP Ticker is loaded.");
+  # queue: start time, server id, counter, time limit
+  my %p = (start => time, id => 0, c => 0, limit => 900); # pending: 15m
+  my %u = (start => time, id => 0, c => 0, limit => 300); # updater:  5m
 
-  # queue -- which address is next in line?
-  my %reset = (start => time, id => 0);
-  my %pending = (%reset, c => 0, limit =>   900); #   900s ~ 15m
-  my %updater = (%reset, c => 0, limit =>  1800); #  1800s ~ 30m
-  my %ut_serv = (%reset, c => 0, limit =>   300); #   300s ~  5m
-  my %oldserv = (%reset, c => 0, limit => 86400); # 86400s ~ 24h
-  
-  my $debug_counter = 0;
-  
-  # go through all servers that need querying
+  # tick through pending list and server list
   my $server_info = AnyEvent->timer (
-    after     => 120, # first give beacons a chance to uplink
-    interval  => 0.2, # 5 addresses per second is fast enough
+    after     => 120, # grace time receiving beacons
+    interval  => 0.2, # ~5 servers/s
     cb        => sub {
-      
-      # after the first full run was completed, reset the counters when loop time expires
-      if (defined $self->{firstrun}) {
-        # reset timer
-        %reset = (start => time, id => 0, c => 0);
-        
-        #
-        # it can happen that a run takes more than the allowed time
-        # in that case, allow more time
-        #
-        
-        # pending
-        if (time - $pending{start} > $pending{limit}) {
-          if ($pending{c} > 0) {
-            # done within defined time, reset
-            %pending = (%pending, %reset);
-          }
-        }
-        
-        # ut servers
-        if (time - $ut_serv{start} > $ut_serv{limit}) {
-          if ($ut_serv{c} > 0) {
-            # done within defined time, reset
-            %ut_serv = (%ut_serv, %reset);
-          }
-        }
-
-        # updater
-        if (time - $updater{start} > $updater{limit}) {
-          if ($updater{c} > 0) {
-            # done within defined time, reset
-            %updater = (%updater, %reset);
-          }
-        }
-        
-        # old servers
-        if (time - $oldserv{start} > $oldserv{limit}) {
-          if ($oldserv{c} > 0) {
-            %oldserv = (%oldserv, %reset);
-          }
-        }
-      }
-
-      #
-      # Check pending beacons
-      #
-
-      # pending beacons/servers (15 seconds grace time)
-      my $n = $self->get_pending(
-        next_id => $pending{id}, 
-        added   => 15,
-        sort    => "id",
-        limit => 1
-      )->[0] if $self->{beacon_checker_enabled};
-      
-      # if next pending server/address exists:
-      if ( $n ) {
-        # next pending id will be > $n
-        $pending{id} = $n->{id};
-
-        # query the server using the heartbeat port provided in the beacon/manual add
-        $self->query_udp_server(
-          $n->{id}, 
-          $n->{ip}, 
-          $n->{heartbeat}, 
-          $n->{secure}, # secure string necessary!
-          1,            # request secure challenge
-        );
-
-        # our work is done for this cycle.
-        return;
+      # reset counters if minimum time before reset passed + list processed
+      if ($self->{firstrun}) {
+        if ($p{c} && time - $p{start} > $p{limit}) { # pending reset
+          %p = (%p, start => time, id => 0, c => 0); }
+        if ($u{c} && time - $u{start} > $u{limit}) { # updater reset
+          %u = (%u, start => time, id => 0, c => 0); }
       }
       
-      # pending are done and is allowed to reset at a later stadium
-      $pending{c}++;
-      
-      
-      #
-      # Query Unreal Tournament 99 (demo) servers for serverstats
-      #
-      
-      # next server in line
-      $n = $self->get_server(
-        next_id => $ut_serv{id},
-        updated => 3600,
-        gamename => "ut",
-        sort    => "id",
-        limit   => 1,
-      )->[0] if $self->{utserver_query_enabled};
-      
-      # if next server/address exists:
-      if ( $n ) {
-        #next pending id will be > $n
-        $ut_serv{id} = $n->{id};
+      # Check pending addresses
+      if ( my $n = $self->get_pending(next_id => $p{id}, limit => 1)->[0] ) {
+        $p{id} = $n->{id}; # next id will be >$n
 
-        # query the server (no secure string)
+        # assign BeaconChecker to query the server for validate, status
         $self->query_udp_server(
-          $n->{id}, 
-          $n->{ip},
-          $n->{port},
-          "", # no secure string necessary
-          2,  # request full status info
+          ip    => $n->{ip},
+          port  => $n->{heartbeat},
+          need_validate => 1,
         );
-
-        # our work is done for this cycle.
         return;
       }
+      $p{c}++; # all pending addresses were processed
       
-      # ut servers are done and is allowed to reset at a later stadium
-      $ut_serv{c}++;
-
-      #
-      # update existing servers (both ut/non-ut)
-      #
-      
-      # next server in line
-      $n = $self->get_server(
-        next_id => $updater{id},
-        updated => 7200,
-        sort    => "id",
-        limit   => 1,
-      )->[0] if $self->{beacon_checker_enabled};
-      
-      # if next server/address exists:
-      if ( $n ) {
-        #next pending id will be > $n
-        $updater{id} = $n->{id};
+      # Update server status
+      if ( my $n = $self->get_server(
+        next_id => $u{id}, 
+        updated => 7200, # count >2h as unresponsive
+        limit => 1
+      )->[0] ) {
+        $u{id} = $n->{id}; # next id will be >$n
 
-        # query the server (no secure string)
+        # assign BeaconChecker to query the server for status (no validate)
         $self->query_udp_server(
-          $n->{id}, 
-          $n->{ip},
-          $n->{port},
-          "", # no secure string necessary
-          0,  # request info
+          ip    => $n->{ip},
+          port  => $n->{port},
         );
-
-        # our work is done for this cycle.
         return;
-      }
-      
-      # updating servers is done and is allowed to reset at a later stadium
-      $updater{c}++;
-      
-      #
-      # Query servers older than 2 hours
-      #
+      }    
+      $u{c}++; # all servers were processed  
       
-      # next server in line
-      $n = $self->get_server(
-        next_id => $oldserv{id},
-        before => 7200,
-        (defined $self->{firstrun}) ? () : (updated => 86400), # FIXME long firstrun time fixed now?
-        sort    => "id",
-        limit   => 1,
-      )->[0] if $self->{beacon_checker_enabled};
-      
-      # if next server/address exists:
-      if ( $n ) {
-        #next old server id will be > $n
-        $oldserv{id} = $n->{id};
-
-        # query the server (no secure string)
-        $self->query_udp_server(
-          $n->{id}, 
-          $n->{ip},
-          $n->{port},
-          "", # no secure string necessary
-          0,  # request info
-        );
-
-        # our work is done for this cycle.
+      # first run complete?
+      if ($self->{firstrun}) {
+        # done. no other actions required
         return;
-      }
-      
-      # old servers are done and is allowed to reset at a later stadium
-      $oldserv{c}++;
-      
-      # and notify about first run being completed
-      if (!defined $self->{firstrun}) {
-        # inform that first run is completed
+      } else {
+        # notify about first run being completed and reset
         my $t = time-$self->{firstruntime};
         my $t_readable = ($t > 60) ? (int($t/60). " minutes ". ($t%60). " seconds") : ($t. " seconds");
-        
-        $self->log("info", "First run completed after $t_readable.");
-        $self->{firstrun} = 0;
-        
-        # reset all counters and follow procedure
-        %reset = (start => time, id => 0, c => 0);
-        %pending = (%pending, %reset);
-        %updater = (%updater, %reset);
-        %ut_serv = (%ut_serv, %reset);
-        %oldserv = (%oldserv, %reset);
-      }
-      
-      # At this point, we are out of server entries. From here on, just count 
-      # down until the cycle is complete and handle new entries while they are 
-      # added to the list.
 
+        $self->log("info", "first run completed after $t_readable");
+        delete $self->{firstruntime};
+        $self->{firstrun} = 1;
+      }
+      # Run complete. Count down until the minimum time has elapsed and handle
+      # new server entries as they are added to the list.
     }
   );
-  
-  # return the timer object to keep it alive outside of this scope
+  # allow object to exist beyond this scope. Objects have ambitions too.
+  $self->log("info", "UDP ticker is loaded");
   return $server_info;
 }
 
diff --git a/lib/MasterServer/UDP/UpLink.pm b/lib/MasterServer/UDP/UpLink.pm
index 01e806e..d523ad0 100755
--- a/lib/MasterServer/UDP/UpLink.pm
+++ b/lib/MasterServer/UDP/UpLink.pm
@@ -2,11 +2,9 @@ package MasterServer::UDP::UpLink;
 
 use strict;
 use warnings;
-use Encode;
 use AnyEvent::Handle::UDP;
 use Socket qw(sockaddr_in inet_ntoa);
 use Exporter 'import';
-
 our @EXPORT = qw| send_heartbeats 
                   do_uplink 
                   process_udp_secure 
@@ -52,7 +50,7 @@ sub do_uplink {
   return unless (defined $ip && defined $port && $port > 0);
   
   # report uplinks to log
-  $self->log("uplink", "Uplink to Masterserver $ip:$port");
+  $self->log("uplink", "uplink to Masterserver $ip:$port");
   
   # connect with UDP server
   my $udp_client; $udp_client = AnyEvent::Handle::UDP->new(
@@ -77,44 +75,35 @@ sub do_uplink {
 ## Note: this replaces the \about\ query in the TCP handler!
 ################################################################################
 sub handle_status_query {
-  my ($self, $udp, $pa, $buf) = @_; 
-
-  # hotfix for one-word queries
-  $buf .= "\\dummy\\";
-  my %r;
-
-  $buf = encode('UTF-8', $buf);
-  $buf =~ s/\n//;
-  $buf =~ s/\\\\/\\undef\\/g; # where to add the +? seperate perl script!
-  $buf =~ s/\\([^\\]+)\\([^\\]+)/$r{$1}=$2/eg;
-
-  # response string
+  # self, handle, packed address, buffer
+  my ($self, $udp, $paddress, $buffer) = @_; 
+  my $rx = $self->data2hashref($buffer);
   my $response = "";
   
   # for compliance, query ids between 0-99
-  $query_id = ($query_id >= 99) ? 1 : ++$query_id;
+  $query_id = ($query_id >= 98) ? 1 : ++$query_id;
   my $sub_id = 1;
   
-  # get database info to present game stats as players, where num_total > 0
+  # get database info to present game stats, where num_total > 0
   my $maxgames = $self->check_cipher_count();
   my $gameinfo = $self->get_game_props(
-                    num_gt => 1, 
-                    sort => "num_total", 
-                    reverse => 1
+    num_gt => 1, 
+    sort => "num_total", 
+    reverse => 1
   );
 
   # secure challenge  
-  if (defined $r{secure}) {
+  if (defined $rx->{secure}) {
     $response .= "\\validate\\"
               .  $self->validate_string(
                     gamename => "333networks",
                     enctype  => 0,
-                    secure   => $r{secure}
+                    secure   => $rx->{secure}
                  );
   }
   
   # basic query
-  if (defined $r{basic} || defined $r{status}) {
+  if (defined $rx->{basic} || defined $rx->{status}) {
     $response .= "\\gamename\\333networks"
               .  "\\gamever\\$self->{short_version}"
               .  "\\location\\0"
@@ -122,37 +111,36 @@ sub handle_status_query {
   }
   
   # info query
-  if (defined $r{info} || defined $r{status}) {
-    $response .= "\\hostname\\$self->{masterserver_hostname}"
+  if (defined $rx->{info} || defined $rx->{status}) {
+    $response .= "\\hostname\\".($self->{masterserver_hostname} || "")
               .  "\\hostport\\$self->{listen_port}"
               .  "\\gametype\\MasterServer"
-              .  "\\numplayers\\". scalar @{$gameinfo}
-              .  "\\maxplayers\\$maxgames"
+              .  "\\mapname\\333networks"
+              .  "\\numplayers\\".(scalar @{$gameinfo} || 0)
+              .  "\\maxplayers\\".($maxgames || 0)
               .  "\\gamemode\\openplaying"
               .  "\\queryid\\$query_id.".$sub_id++;
   }
   
   # rules query
-  if (defined $r{rules} || defined $r{status}) {
-    $response .= "\\mutators\\333networks synchronization, master applet synchronization"
-              .  "\\AdminName\\$self->{masterserver_name}"
-              .  "\\AdminEMail\\$self->{masterserver_contact}"
+  if (defined $rx->{rules} || defined $rx->{status}) {
+    $response .= "\\mutators\\333networks synchronization, UCC Master applet synchronization, Display Stats As Players"
+              .  "\\AdminName\\".($self->{masterserver_name} || "")
+              .  "\\AdminEMail\\".($self->{masterserver_contact} || "")
               .  "\\queryid\\$query_id.".$sub_id++;
   }
   
   # players query
-  if (defined $r{players} || defined $r{status}) {
-    # list game stats as if they were players, with game description as 
-    # player_$, gamename as skin_$, total servers as frags_$ and number of 
-    # direct uplinks as deaths_$
+  if (defined $rx->{players} || defined $rx->{status}) {
+    # list game stats as if they were players. let the client figure out how
+    # to list this information on their website (hint: that's us)
     my $c = 0;
-
     foreach my $p (@{$gameinfo}) {
-      $c++; # count players
-      $response .= "\\player_$c\\$p->{description}"
-                .  "\\skin_$c\\$p->{gamename}"
-                .  "\\frags_$c\\$p->{num_total}"
-                .  "\\deaths_$c\\$p->{num_uplink}";
+      $response .= "\\player_$c\\".($p->{description} || "")
+                .  "\\team_$c\\"  .($p->{gamename}    || "")
+                .  "\\skin_$c\\"  .($p->{num_total}   || 0) . " total"
+                .  "\\mesh_$c\\"  .($p->{num_uplink}  || 0) . " direct";
+      $c++; # start with player_0, increment
     }
     $response .= "\\queryid\\$query_id.".$sub_id++;
   }
@@ -163,10 +151,10 @@ sub handle_status_query {
   # split the response in chunks of 512 bytes and send
   while (length $response > 512) {
     my $chunk = substr $response, 0, 512, '';
-    $udp->push_send($chunk, $pa);
+    $udp->push_send($chunk, $paddress);
   }
   # last <512 chunk
-  $udp->push_send($response, $pa);
+  $udp->push_send($response, $paddress);
 }
 
 1;