//Source file: D:\\Projects\\src\\yi\\iu\\imgdb\\learn\\StatTestRst.java

package yi.iu.imgdb.learn;

import yi.ds.*;
import java.util.*;
import yi.ds.hash.*;
import org.jmat.gui.*;
import org.jmat.data.*;
import java.io.*;
import yi.util.io.output.*;
import yi.util.io.input.*;
import yi.util.config.*;
import yi.util.io.*;
import yi.util.data.*;
import yi.iu.imgdb.desc.*;
import java.text.*;
import yi.ds.sort.*;

/**
 * A hash map with object names as keys.  The values are the hash maps with image
 * names as keys and corresponding scores or probabilities as values.
 */
public class StatTestRst
    extends HashMapHashMap
    implements Serializable {

  static final long serialVersionUID = -7784333788597445604L;

  /**
   * The higher the score, the more likely a positive instance
   */
  protected boolean m_higherIsBetter = true;

  public StatTestRst(boolean higher_is_better) {
    m_higherIsBetter = higher_is_better;
  }

  public StatTestRst() {
    this(true);
  }

  public void putScore(Object obj, Object img, double score) {
    put(obj, img, new Double(score));
  }

  public double getScore(Object obj, Object img) {
    if (get(obj,img)==null) {
      System.out.println("can't get score for " + obj + " for " + img);
      return 0;
    }
    return ( (Double) get(obj, img)).doubleValue();
  }

  /**
   * Generate the right- positive curve data.  The x axis of the curve is the
   * right positive ratio (right positive number / total positive case number)
   * and the y axis is the false positive ratio (false positive number / total
   * negative case number).  The more the curve is close to the bottom right
   * corner, the better the test result.  An entry may look like:
   * "tree" -> {{.1,0},{0.2,0},{.3,.1}, ...}
   * @param data_center
   * @return
   */
  public HashMap rpcAnalysis(ImgDataCenter data_center) {
    // the result
    HashMap rpcs = new HashMap();

    // analysis each object
    List objects = new ArrayList(keySet());
    Collections.sort(objects);
    for (Iterator i = objects.iterator(); i.hasNext(); ) {
      Object the_obj = i.next();
//      System.out.println("----------------------");
//      System.out.println(the_obj.toString() + ":");
      DoubleScoreHashMap one_obj_score = new DoubleScoreHashMap(
          (HashMap) get(the_obj));

      HashSet pos_set = new HashSet();
      for (Iterator j = one_obj_score.keySet().iterator(); j.hasNext();) {
        String the_img = (String) j.next();
        LinkedList objs = data_center.getImgDescriptions(the_img);
        if (objs!=null && objs.contains(the_obj)) {
          pos_set.add(the_img);
        }
      }
      double[][] rpc = rpcAnalysis(one_obj_score,
                                pos_set, m_higherIsBetter);
      rpcs.put(the_obj, rpc);
    }

    return rpcs;
  }

  public HashMap rpcAnalysis(ImgDescManager  idm) {
    // the result
    HashMap rpcs = new HashMap();

    // analysis each object
    List objects = new ArrayList(keySet());
    Collections.sort(objects);
    for (Iterator i = objects.iterator(); i.hasNext(); ) {
      Object the_obj = i.next();
//      System.out.println("----------------------");
//      System.out.println(the_obj.toString() + ":");
      DoubleScoreHashMap one_obj_score = new DoubleScoreHashMap(
          (HashMap) get(the_obj));

      HashSet pos_set = new HashSet();
      for (Iterator j = one_obj_score.keySet().iterator(); j.hasNext();) {
        String the_img = (String) j.next();
        HashSet objs = idm.getDesc(the_img);
        if (objs!=null && objs.contains(the_obj)) {
          pos_set.add(the_img);
        }
      }
      double[][] rpc = rpcAnalysis(one_obj_score,
                                pos_set, m_higherIsBetter);
      rpcs.put(the_obj, rpc);
    }

    return rpcs;
  }

  /**
   * Precision and Recall analysis
   * @param data_center
   * @return
   */
  public HashMap precisionAndRecallAnalysis(ImgDescManager idm) {

    double default_threshold = 0.5;
    double threshold = default_threshold;

      // the result
    HashMap rpcs = new HashMap();

    // analysis each object
    System.out.println("precision and recall");
    List objects = new ArrayList(keySet());
    Collections.sort(objects);
//for (int t=-10; t<=100; t++) {
//      threshold = t*0.01;
//      int good_obj_count = 0;
      for (Iterator i = objects.iterator(); i.hasNext(); ) {
        Object the_obj = i.next();
//      if (this instanceof StatTestRstWithVerErr) {
//        if (((StatTestRstWithVerErr)this).getThresholds().get(the_obj) != null) {
//        threshold = ((StatTestRstWithVerErr)this).getThresholds().getScore(the_obj);
//        System.out.println(the_obj.toString() + " threshold = " + threshold);
//        } else {
//          threshold = default_threshold;
//        }
//      }

//      System.out.println("----------------------");
//      System.out.println(the_obj.toString() + ":");
        DoubleScoreHashMap one_obj_score = new DoubleScoreHashMap(
            (HashMap) get(the_obj));

        HashSet pos_set = new HashSet();
        for (Iterator j = one_obj_score.keySet().iterator(); j.hasNext(); ) {
          String the_img = (String) j.next();
          HashSet objs = idm.getDesc(the_img);
          if (objs != null && objs.contains(the_obj)) {
            pos_set.add(the_img);
          }
        }

        // sort the images according to their score
        ArrayList the_images = new ArrayList(one_obj_score.keySet());
        Collections.sort(the_images, one_obj_score);
        // sort from positive to less positive
        Collections.reverse(the_images);

        // find those scores greater than 0.5
        int true_positive = 0, total_return = 1;
        double best_precision = 0, best_recall = 0;
        double[] pres = new double[the_images.size()];
        double[] recs = new double[the_images.size()];
        for (Iterator j = the_images.iterator(); j.hasNext(); total_return++) {
          Object img = j.next();
          double score = one_obj_score.getScore(img);
          if (pos_set.contains(img))
            true_positive++;
          double precision = (double) true_positive / (double) total_return;
          double recall = (double) true_positive / (double) pos_set.size();
//        if (precision + recall > best_precision + best_recall) {
          if (score >= threshold) {
            best_precision = precision;
            best_recall = recall;
          }
          pres[total_return - 1] = precision;
          recs[total_return - 1] = recall;
        }

        rpcs.put(the_obj, new PrecisionAndRecall(best_precision, best_recall));

//      OutputResult.print(System.out, pres, pres.length, 1);
//      OutputResult.print(System.out, recs, recs.length, 1);
        System.out.println(the_obj.toString() + "\t" + best_precision + "\t" +
                           best_recall);
//        if (best_precision >= 0.15 && best_recall >= 0.4) {
//          good_obj_count++;
//        }
//      }
//      System.out.println("threshold=\t"+threshold+"\tcount=\t"+good_obj_count);
    }

    return rpcs;
  }

  public HashMap precisionAndRecallAnalysis(ImgDataCenter data_center) {

    double default_threshold = 0.5;
    double threshold = default_threshold;

      // the result
    HashMap rpcs = new HashMap();

    // analysis each object
    System.out.println("precision and recall");
    List objects = new ArrayList(keySet());
    Collections.sort(objects);
//for (int t=-10; t<=100; t++) {
//      threshold = t*0.01;
//      int good_obj_count = 0;
      for (Iterator i = objects.iterator(); i.hasNext(); ) {
        Object the_obj = i.next();
//      if (this instanceof StatTestRstWithVerErr) {
//        if (((StatTestRstWithVerErr)this).getThresholds().get(the_obj) != null) {
//        threshold = ((StatTestRstWithVerErr)this).getThresholds().getScore(the_obj);
//        System.out.println(the_obj.toString() + " threshold = " + threshold);
//        } else {
//          threshold = default_threshold;
//        }
//      }

//      System.out.println("----------------------");
//      System.out.println(the_obj.toString() + ":");
        DoubleScoreHashMap one_obj_score = new DoubleScoreHashMap(
            (HashMap) get(the_obj));

        HashSet pos_set = new HashSet();
        for (Iterator j = one_obj_score.keySet().iterator(); j.hasNext(); ) {
          String the_img = (String) j.next();
          LinkedList objs = data_center.getImgDescriptions(the_img);
          if (objs != null && objs.contains(the_obj)) {
            pos_set.add(the_img);
          }
        }

        // sort the images according to their score
        ArrayList the_images = new ArrayList(one_obj_score.keySet());
        Collections.sort(the_images, one_obj_score);
        // sort from positive to less positive
        Collections.reverse(the_images);

        // find those scores greater than 0.5
        int true_positive = 0, total_return = 1;
        double best_precision = 0, best_recall = 0;
        double[] pres = new double[the_images.size()];
        double[] recs = new double[the_images.size()];
        for (Iterator j = the_images.iterator(); j.hasNext(); total_return++) {
          Object img = j.next();
          double score = one_obj_score.getScore(img);
          if (pos_set.contains(img))
            true_positive++;
          double precision = (double) true_positive / (double) total_return;
          double recall = (double) true_positive / (double) pos_set.size();
//        if (precision + recall > best_precision + best_recall) {
          if (score >= threshold) {
            best_precision = precision;
            best_recall = recall;
          }
          pres[total_return - 1] = precision;
          recs[total_return - 1] = recall;
        }

        rpcs.put(the_obj, new PrecisionAndRecall(best_precision, best_recall));

//      OutputResult.print(System.out, pres, pres.length, 1);
//      OutputResult.print(System.out, recs, recs.length, 1);
        System.out.println(the_obj.toString() + "\t" + best_precision + "\t" +
                           best_recall);
//        if (best_precision >= 0.15 && best_recall >= 0.4) {
//          good_obj_count++;
//        }
//      }
//      System.out.println("threshold=\t"+threshold+"\tcount=\t"+good_obj_count);
    }

    return rpcs;
  }

  /**
   * The image can be classified to one category according to its score on
   * the available objects (concepts).  Return the image->object map
   * @param scores return the score for the classified object.  Pass in "null"
   *        if you are not interested
   * @return
   */
  public LinkedListHashMap classify(DoubleScoreHashMap scores) {

    // save the highest score (for the classified object)
    if (scores == null) scores = new DoubleScoreHashMap();

    LinkedListHashMap classified_map = new LinkedListHashMap();

    // analysis each object
    List objects = new ArrayList(keySet());
    Collections.sort(objects);
    for (Iterator i = objects.iterator(); i.hasNext(); ) {
      Object the_obj = i.next();
      DoubleScoreHashMap one_obj_score = new DoubleScoreHashMap(
          (HashMap) get(the_obj));

      for (Iterator j = one_obj_score.keySet().iterator(); j.hasNext();) {
        String the_img = (String) j.next();
        double score = one_obj_score.getScore(the_img);

        if (scores.putScoreWhenHigher(the_img, score)) {
          LinkedList obj_list = new LinkedList();
          obj_list.add(the_obj);
          classified_map.put(the_img, obj_list);
        }
      }
    }

    return classified_map;
  }

  /**
   * Analysis the classification result.  Each image was classified to one
   * object as specified in "classified_map".   Analysis that result, form a
   * matrix.  Each entry in that matrix is how many instances was classified
   * into that concept.  The row (first dim) is the real object index.  The
   * column (second dim) is the classified object.  Eg. rst[0][1] = 4 means
   * 4 images of concept (object) 0 was classified into concept 1.
   * @param dc
   * @return
   */
  public int[][] classifyAnalysis(ImgDataCenter data_center) {
    LinkedListHashMap classified_map = classify(null);

    // analysis each object
    List objects = new ArrayList(data_center.getObjectList());
    Collections.sort(objects);

    int num_obj = objects.size();
    int[][] count = new int[num_obj][num_obj];

    // make a object -> index map to increase the speed
    HashMap obj2ind = new HashMap();
    int i_index = 0;
    for (Iterator i = objects.iterator(); i.hasNext(); i_index++) {
      Object obj = i.next();
      obj2ind.put(obj, new Integer(i_index));
      System.out.println(obj.toString() + " -> " + i_index);
    }

    for (Iterator i = classified_map.keySet().iterator(); i.hasNext(); i_index++) {
      String img = (String)i.next();
      // classified object list
      LinkedList objs = classified_map.getList(img);

      // real labels (object list)
      LinkedList labels = data_center.getImgDescriptions(img);

      for (Iterator j= labels.iterator(); j.hasNext();) {
        String label = (String)j.next();
        int label_ind = ((Integer)obj2ind.get(label)).intValue();

        for (Iterator k= objs.iterator(); k.hasNext();) {
          String obj = (String)k.next();
          int obj_ind = ((Integer)obj2ind.get(obj)).intValue();
          count[label_ind][obj_ind] ++;
        }

      }

    }
    return count;
  }


  /**
   * Analysis the classification result.  Each image was classified to one
   * object as specified in "classified_map".   Analysis that result, form a
   * matrix.  Each entry in that matrix is how many instances was classified
   * into that concept.  The row (first dim) is the real object index.  The
   * column (second dim) is the classified object.  Eg. rst[0][1] = 4 means
   * 4 images of concept (object) 0 was classified into concept 1.
   * @param dc
   * @return
   */
  public int[][] classifyAnalysis(ImgDescManager idm) {
    LinkedListHashMap classified_map = classify(null);

    // analysis each object
    List objects = new ArrayList(keySet());
    Collections.sort(objects);

    int num_obj = objects.size();
    int[][] count = new int[num_obj][num_obj];

    // make a object -> index map to increase the speed
    HashMap obj2ind = new HashMap();
    int i_index = 0;
    for (Iterator i = objects.iterator(); i.hasNext(); i_index++) {
      Object obj = i.next();
      obj2ind.put(obj, new Integer(i_index));
      System.out.println(obj.toString() + " -> " + i_index);
    }

    for (Iterator i = classified_map.keySet().iterator(); i.hasNext(); i_index++) {
      String img = (String)i.next();
      // classified object list
      LinkedList objs = classified_map.getList(img);

      // real labels (object list)
      HashSet labels = idm.getDesc(img);
      for (Iterator j= labels.iterator(); j.hasNext();) {
        String label = (String)j.next();
        if (!obj2ind.containsKey(label)) continue; // the results may not contain the
                                                   // test on all the object
        int label_ind = ((Integer)obj2ind.get(label)).intValue();

        for (Iterator k= objs.iterator(); k.hasNext();) {
          String obj = (String)k.next();
          int obj_ind = ((Integer)obj2ind.get(obj)).intValue();
          count[label_ind][obj_ind] ++;
        }

      }

    }

    i_index=0;
    for (Iterator i = objects.iterator(); i.hasNext(); i_index++) {
      Object obj = i.next();
      System.out.println(obj.toString() + "\t" + count[i_index][i_index]);
    }

    return count;
  }

  /**
   * Analysis the classification result.  Each image was classified to one
   * object as specified in "classified_map".   Analysis that result, form a
   * matrix.  Each entry in that matrix is how many instances was classified
   * into that concept.  The row (first dim) is the real object index.  The
   * column (second dim) is the classified object.  Eg. rst[0][1] = 4 means
   * 4 images of concept (object) 0 was classified into concept 1.
   * @param dc
   * @return
   */
  public void labellingAnalysis(ImgDescManager idm, int topn) {

    // analysis each object
    List objects = new ArrayList(keySet());
    Collections.sort(objects);
    int num_obj = objects.size();

    // images
    List images = new ArrayList(idm.getFiles());
    Collections.sort(images);

    // make a object -> index map to increase the speed
    HashMap obj2ind = new HashMap();
    int i_index = 0;
    for (Iterator i = objects.iterator(); i.hasNext(); i_index++) {
      Object obj = i.next();
      obj2ind.put(obj, new Integer(i_index));
      System.out.println(obj.toString() + " -> " + i_index);
    }
    double[] scores = new double[num_obj];
    int[] label_flag = new int[num_obj];
    int[] sum = new int[topn];
    i_index = 0;
    for (Iterator i = images.iterator(); i.hasNext(); i_index++) {
      String img = (String)i.next();
      Arrays.fill(scores, 0);
      for (Iterator o = objects.iterator(); o.hasNext(); ) {
        String obj = (String) o.next();
        DoubleScoreHashMap one_obj_score = new DoubleScoreHashMap(
            (HashMap) get(obj));
        int label_ind = ( (Integer) obj2ind.get(obj)).intValue();
        scores[label_ind] = one_obj_score.getScore(img);
      }
      int[] sorted_index = SortUtil.sort(scores);

      // real labels (object list)
      HashSet labels = idm.getDesc(img);
      System.out.print(img);
      Arrays.fill(label_flag, 0);
      for (int top = 1; top<= topn; top++) {
        label_flag[sorted_index[num_obj - top]] = 1;
        int found = 0;
        for (Iterator j = labels.iterator(); j.hasNext(); ) {
          String label = (String) j.next();
          if (!obj2ind.containsKey(label))
            continue; // the results may not contain the
          // test on all the object
          int label_ind = ( (Integer) obj2ind.get(label)).intValue();
          if (label_flag[label_ind] > 0) found++;
        }
        System.out.print("\t" + scores[sorted_index[num_obj - top]] );
        System.out.print("\t" + found);
        sum[top-1] += found;
      }
      System.out.println();
    }
    OutputResult.print(System.out, sum);

  }


  /**
   * Generate the right- positive curve data. (for ROC chart)
   * @param object_score_map the mapping between objects and their scores
   * @param pos_set the positive object set
   * @return
   */
  public static double[][] rpcAnalysis(DoubleScoreHashMap obj_score_map,
                                HashSet pos_set, boolean high_is_positive) {

    // sort the images according to their score
    ArrayList the_images = new ArrayList(obj_score_map.keySet());
    Collections.sort(the_images, obj_score_map);
    // sort from more positive to less positive
    if (high_is_positive) {
      Collections.reverse(the_images);
    }

    // consider thresholds which only passes the top n images.
    // in the passed images, there are right positives and false positives
    // get the numbers
    int[] right_pos = new int[the_images.size() + 1],
        mis_neg = new int[the_images.size() + 1]; // right positive, mis-classified neg

    right_pos[0] = 0;
    mis_neg[0] = 0; // the first threshold, no one passes
    int pos = 0, neg = 0; // total number of pos and neg
    int index = 0;
    for (Iterator j = the_images.iterator(); j.hasNext(); index++) {
      Object obj = j.next();
      double score = obj_score_map.getScore(obj);

      boolean contains = pos_set.contains(obj);
      if (contains) { // right positive
        pos++;
        right_pos[index + 1] = right_pos[index] + 1; // incread the threshold, one postive just passed
        mis_neg[index + 1] = mis_neg[index];
      }
      else { // false positve
        neg++;
        mis_neg[index + 1] = mis_neg[index] + 1; // incread the threshold, one negative just passed
        right_pos[index + 1] = right_pos[index];
      }
    }
    assert (pos == pos_set.size());
    double[][] rpc = new double[right_pos.length][2]; // receiver operating curve
    for (int j = 0; j < right_pos.length; j++) {
      rpc[j][0] = (pos != 0 && neg != 0) ? (right_pos[j] / (double) pos) : 0;
      rpc[j][1] = (pos != 0 && neg != 0) ? (mis_neg[j] / (double) neg) : 0;
    }
    return rpc;
  }

  public static double rpcEquallPoint(DoubleScoreHashMap obj_score_map,
                                HashSet pos_set, boolean high_is_positive) {

    // sort the images according to their score
    ArrayList the_images = new ArrayList(obj_score_map.keySet());
    Collections.sort(the_images, obj_score_map);
    // sort from more positive to less positive
    if (high_is_positive) {
      Collections.reverse(the_images);
    }

    // consider thresholds which only passes the top n images.
    // in the passed images, there are right positives and false positives
    // get the numbers
    int[] right_pos = new int[the_images.size() + 1],
        mis_neg = new int[the_images.size() + 1]; // right positive, mis-classified neg

    right_pos[0] = 0;
    mis_neg[0] = 0; // the first threshold, no one passes
    int pos = 0, neg = 0; // total number of pos and neg
    int index = 0;
    double score = 0;
    double last_pos_score = 1;
    for (Iterator j = the_images.iterator(); j.hasNext(); index++) {
      Object obj = j.next();
      score = obj_score_map.getScore(obj);

      boolean contains = pos_set.contains(obj);
      if (contains) { // right positive
        pos++;
        right_pos[index + 1] = right_pos[index] + 1; // incread the threshold, one postive just passed
        mis_neg[index + 1] = mis_neg[index];
      }
      else { // false positve
        neg++;
        mis_neg[index + 1] = mis_neg[index] + 1; // incread the threshold, one negative just passed
        right_pos[index + 1] = right_pos[index];
      }

      double true_pos = right_pos[index + 1] / (double) pos_set.size();
      double false_pos = mis_neg[index + 1] / (double) (obj_score_map.size() - pos_set.size());
      if (true_pos >= 1-false_pos) {
        System.out.println("scores = " + last_pos_score + ", " + score);
        System.out.println("true_pos = " + true_pos + ", false_pos = " + false_pos);
        score = (score + last_pos_score)/2;
        System.out.println("threshold = " + score);
        break;
      }
      if (contains) last_pos_score = score;
    }
    return score;
  }


  /**
   * Output the sorted object in html files.  Each file corresponding to one
   * object
   * @param html_prefix
   * @return
   */
  public void outputSortedHtml(ImgDataCenter data_center, String html_prefix,
                               String img_path, String rst_path) {
    // the result
    HashMap rpcs = new HashMap();

    // analysis each object
    List objects = new ArrayList(keySet());
    Collections.sort(objects);
    for (Iterator i = objects.iterator(); i.hasNext(); ) {
      Object the_obj = i.next();
      PrintStream out = OutputResult.getPrintStream(html_prefix + "." + the_obj +
          ".html");

      out.println("<html><body>");
      out.println("<h3>" + the_obj.toString() + "</h3>");
      out.println("<table>");
      DoubleScoreHashMap one_obj_score = new DoubleScoreHashMap(
          (HashMap) get(the_obj));
      // sort the images according to their score
      ArrayList the_images = new ArrayList(one_obj_score.keySet());
      Collections.sort(the_images, one_obj_score);
      // sort from more positive to less positive
      if (m_higherIsBetter) {
        Collections.reverse(the_images);
      }

      // find the image feaure used
      HashSet features = new HashSet(data_center.getFeatureList());

      int index = 0;
      for (Iterator j = the_images.iterator(); j.hasNext(); index++) {
        String the_img = (String) j.next();
        double score = one_obj_score.getScore(the_img);
        LinkedList objs = data_center.getImgDescriptions(the_img);
        boolean contains = false;
        if (objs != null && objs.contains(the_obj)) { // right positive
          contains = true;
        }

        if (index != 0)
          out.println("</tr>");
        out.println("<tr>");
        //** @todo change to use groundtruth.thumb */
        String the_img_thumb = yi.iu.imgdb.desc.ImgDescManager.toThumbImageName(the_img);
        String scale = ".s256";
        out.print("<td>");
        out.println("<font size=\"-2\" color=" +
                    (contains ? "#FF0000" : "#00FF00") + ">" +
                    score + "</font><br>");
        out.print("<a HREF=" + img_path + the_img.toString() +
                  "><img SRC=" + img_path + the_img_thumb.toString() +
                  " ALT=" + the_img_thumb.toString() + "></a>");

        if (features.contains("yi.iu.processor.color.reg.ColorLabRgnFtsPxlVal")) {
          out.print("<a HREF=" + rst_path + the_img.toString() + scale +
                    ".color.rgn.gif" +
                    "><img SRC=" + rst_path + the_img.toString() + scale +
                    ".color.rgn.gif" +
                    " ALT=" + the_img.toString() + scale + ".color.rgn.gif" +
                    "></a>");
            out.print("<a HREF=" + rst_path + the_img.toString() + scale +
                      ".color.rgnc.gif" +
                      "><img SRC=" + rst_path + the_img.toString() + scale +
                      ".color.rgnc.gif" +
                      " ALT=" + the_img.toString() + scale + ".color.rgnc.gif" +
                      "></a>");
        }

        if (features.contains("yi.iu.processor.texture.reg.TextureLabRgnFtsPxlVal")) {
            out.print("<a HREF=" + rst_path + the_img.toString() + scale +
                      ".text.rgn.gif" +
                      "><img SRC=" + rst_path + the_img.toString() + scale +
                      ".text.rgn.gif" +
                      " ALT=" + the_img.toString() + scale + ".text.rgn.gif" +
                      "></a>");
        }

        if (features.contains("yi.iu.processor.color.reg.CIELabLabeledRgnFts")) {
            out.print("<a HREF=" + rst_path + the_img.toString() + scale +
                      ".dc.rgn.gif" +
                      "><img SRC=" + rst_path + the_img.toString() + scale +
                      ".dc.rgn.gif" +
                      " ALT=" + the_img.toString() + scale + ".dc.rgn.gif" +
                      "></a>");
        }

        if (features.contains("yi.iu.region.LabeledRgnFtsWithPxlValue")) {
            out.print("<a HREF=" + rst_path + the_img.toString() + scale +
                      ".ct.rgn.gif" +
                      "><img SRC=" + rst_path + the_img.toString() + scale +
                      ".ct.rgn.gif" +
                      " ALT=" + the_img.toString() + scale + ".ct.rgn.gif" +
                      "></a>");
        }

        if (features.contains("yi.iu.processor.clc.LineClusterSets")) {
            out.print("<a HREF=" + rst_path + the_img.toString() +
                      ".CLC_Spatial.gif" +
                      "><img SRC=" + rst_path + the_img.toString() +
                      ".CLC_Spatial.gif" +
                      " WIDTH=256 ALT=" + the_img.toString() +
                      ".CLC_Spatial.gif" +
                      "></a>");
        }

        out.println("</td>");
//        if (index % 5 == 0 && index != 0)
//          out.println("</tr>");
//        if (index % 5 == 0)
//          out.println("<tr>");
//        //** @todo change to use groundtruth.thumb */
//        the_img = the_img.replaceFirst("groundtruth", "groundtruth.thumb");
//        out.println("<td><a HREF=" +
//                    the_img.toString() + "><img SRC=" + the_img.toString() +
//                    " ALT=" + the_img.toString() + "><br>" +
//                    "<font size=\"-2\" color=" +
//                    (contains ? "#FF0000" : "#00FF00") + ">" +
//                    score + "</font><br></a></td>");
      }
      out.println("</tr></table></body></html>");
      out.close();
    }
  }


  /**
   * Output the sorted object in html files.  Each file corresponding to one
   * object
   * @param html_prefix
   * @return
   */
  public void outputSortedHtml(ImgDescManager idm, String html_prefix,
                               String img_path, String rst_path) {
    // the result
    HashMap rpcs = new HashMap();

    // analysis each object
    List objects = new ArrayList(keySet());
    Collections.sort(objects);
    for (Iterator i = objects.iterator(); i.hasNext(); ) {
      Object the_obj = i.next();
      PrintStream out = OutputResult.getPrintStream(html_prefix + "." + the_obj +
          ".html");

      out.println("<html><body>");
      out.println("<h3>" + the_obj.toString() + "</h3>");
      out.println("<table>");
      DoubleScoreHashMap one_obj_score = new DoubleScoreHashMap(
          (HashMap) get(the_obj));
      // sort the images according to their score
      ArrayList the_images = new ArrayList(one_obj_score.keySet());
      Collections.sort(the_images, one_obj_score);
      // sort from more positive to less positive
      if (m_higherIsBetter) {
        Collections.reverse(the_images);
      }

//      // find the image feaure used
//      HashSet features = new HashSet(data_center.getFeatureList());

      int index = 0;
      for (Iterator j = the_images.iterator(); j.hasNext(); index++) {
        String the_img = (String) j.next();
        double score = one_obj_score.getScore(the_img);
        HashSet objs = idm.getDesc(the_img);
        boolean contains = false;
        if (objs != null && objs.contains(the_obj)) { // right positive
          contains = true;
        }

        if (index != 0)
          out.println("</tr>");
        out.println("<tr>");
        //** @todo change to use groundtruth.thumb */
        String the_img_thumb = yi.iu.imgdb.desc.ImgDescManager.toThumbImageName(the_img);
        String scale = ".s256";
        out.print("<td>");
        out.println("<font size=\"-2\" color=" +
                    (contains ? "#FF0000" : "#00FF00") + ">" +
                    score + "</font><br>");
        out.print("<a HREF=" + img_path + the_img.toString() +
                  "><img SRC=" + img_path + the_img_thumb.toString() +
                  " ALT=" + the_img_thumb.toString() + "></a>");

//        if (features.contains("yi.iu.processor.color.reg.ColorLabRgnFtsPxlVal")) {
//          out.print("<a HREF=" + rst_path + the_img.toString() + scale +
//                    ".color.rgn.gif" +
//                    "><img SRC=" + rst_path + the_img.toString() + scale +
//                    ".color.rgn.gif" +
//                    " ALT=" + the_img.toString() + scale + ".color.rgn.gif" +
//                    "></a>");
//            out.print("<a HREF=" + rst_path + the_img.toString() + scale +
//                      ".color.rgnc.gif" +
//                      "><img SRC=" + rst_path + the_img.toString() + scale +
//                      ".color.rgnc.gif" +
//                      " ALT=" + the_img.toString() + scale + ".color.rgnc.gif" +
//                      "></a>");
//        }
//
//        if (features.contains("yi.iu.processor.texture.reg.TextureLabRgnFtsPxlVal")) {
//            out.print("<a HREF=" + rst_path + the_img.toString() + scale +
//                      ".text.rgn.gif" +
//                      "><img SRC=" + rst_path + the_img.toString() + scale +
//                      ".text.rgn.gif" +
//                      " ALT=" + the_img.toString() + scale + ".text.rgn.gif" +
//                      "></a>");
//        }
//
//        if (features.contains("yi.iu.processor.color.reg.CIELabLabeledRgnFts")) {
//            out.print("<a HREF=" + rst_path + the_img.toString() + scale +
//                      ".dc.rgn.gif" +
//                      "><img SRC=" + rst_path + the_img.toString() + scale +
//                      ".dc.rgn.gif" +
//                      " ALT=" + the_img.toString() + scale + ".dc.rgn.gif" +
//                      "></a>");
//        }
//
//        if (features.contains("yi.iu.region.LabeledRgnFtsWithPxlValue")) {
//            out.print("<a HREF=" + rst_path + the_img.toString() + scale +
//                      ".ct.rgn.gif" +
//                      "><img SRC=" + rst_path + the_img.toString() + scale +
//                      ".ct.rgn.gif" +
//                      " ALT=" + the_img.toString() + scale + ".ct.rgn.gif" +
//                      "></a>");
//        }
//
//        if (features.contains("yi.iu.processor.clc.LineClusterSets")) {
//            out.print("<a HREF=" + rst_path + the_img.toString() +
//                      ".CLC_Spatial.gif" +
//                      "><img SRC=" + rst_path + the_img.toString() +
//                      ".CLC_Spatial.gif" +
//                      " WIDTH=256 ALT=" + the_img.toString() +
//                      ".CLC_Spatial.gif" +
//                      "></a>");
//        }

        out.println("</td>");
//        if (index % 5 == 0 && index != 0)
//          out.println("</tr>");
//        if (index % 5 == 0)
//          out.println("<tr>");
//        //** @todo change to use groundtruth.thumb */
//        the_img = the_img.replaceFirst("groundtruth", "groundtruth.thumb");
//        out.println("<td><a HREF=" +
//                    the_img.toString() + "><img SRC=" + the_img.toString() +
//                    " ALT=" + the_img.toString() + "><br>" +
//                    "<font size=\"-2\" color=" +
//                    (contains ? "#FF0000" : "#00FF00") + ">" +
//                    score + "</font><br></a></td>");
      }
      out.println("</tr></table></body></html>");
      out.close();
    }
  }

  /**
   * Return the ROC charts (each element in the returned HashMap is a double[][]
   * each double[i][0] is the false positive rate and each double[i][1] is
   * true positive rate
   * @param rpc
   * @return
   */
  public static HashMap rpc2roc(HashMap rpcs) {
    HashMap rocs = new HashMap();
    for (Iterator i = rpcs.keySet().iterator(); i.hasNext(); ) {
      Object obj = i.next();
      double[][] rpc = (double[][]) rpcs.get(obj);
      // change to roc terminology
      double[][] roc = rpc2roc(rpc);
      rocs.put(obj, roc);
    }
    return rocs;
  }

  /**
   * In rpc (right positive curve), x is right positive and y is false positive,
   * but in roc, the opposite
   * @param rpc
   * @return
   */
  public static double[][] rpc2roc(double[][] rpc) {
    // change to roc terminology
    double[][] roc = new double[rpc.length][2];
    for (int j=0; j<roc.length; j++) {
      roc[j][0] = rpc[j][1];
      roc[j][1] = rpc[j][0];
    }
    return roc;
  }

  public static double getROCScore(double[][] roc) {
    double area = 0;
    for (int i=1; i<roc.length; i++) {
      double delta_x = roc[i][0] - roc[i-1][0];
      double average_high = (roc[i][1] + roc[i-1][1]) / 2;
      area += delta_x * average_high;
    }
    return area;
  }

  public static DoubleScoreHashMap getROCScores(HashMap rpcs) {
    HashMap rocs = rpc2roc(rpcs);
    DoubleScoreHashMap scores = new DoubleScoreHashMap();
    for (Iterator i = rocs.keySet().iterator(); i.hasNext(); ) {
      Object obj = i.next();
      double[][] roc = (double[][]) rocs.get(obj);
      double score = getROCScore(roc);
      scores.putScore(obj, score);
    }
    return scores;
  }

  public static Plot2DPanel drawROCs(HashMap rpcs) {
    // the draw panel
    Plot2DPanel pp = new Plot2DPanel();

    // analysis each object
    HashMap rocs = rpc2roc(rpcs);
    List objects = new ArrayList(rocs.keySet());
    Collections.sort(objects);
    for (Iterator i = objects.iterator(); i.hasNext(); ) {
      Object obj = i.next();
      System.out.println(obj);
      double[][] roc = (double[][]) rocs.get(obj);
      double[][] roc_flip = Conversion.swapXandY(roc, roc[0].length, roc.length);
      OutputResult.print(System.out, roc_flip);
      Matrix m = new Matrix(roc);
      m.toPlot2DPanel(pp, obj.toString(), "LINE");
    }
    pp.setAxesLabels("Flase Positive Rate", "True Positive Rate");
    pp.setPreferredSize(new java.awt.Dimension (600, 600));

    ExistablePlotViewer viewer = new ExistablePlotViewer("ROC", pp);

    return pp;

  }

  public static void outputROC(HashMap rpcs) {

    // analysis each object
    HashMap rocs = rpc2roc(rpcs);
    List objects = new ArrayList(rocs.keySet());
    Collections.sort(objects);
    double[][] roc_data = new double[objects.size() * 2][];
    int i_index = 0;
    for (Iterator i = objects.iterator(); i.hasNext(); i_index++) {
      Object obj = i.next();
      System.out.print(obj + "\t\t");
      double[][] roc = (double[][]) rocs.get(obj);
      double[][] roc_flip = Conversion.swapXandY(roc, roc[0].length, roc.length);
      roc_data[2*i_index] = roc_flip[0];
      roc_data[2*i_index+1] = roc_flip[1];
    }
    System.out.println();
    double[][] roc_data_flip = Conversion.swapXandY(roc_data, roc_data[0].length, roc_data.length);
    OutputResult.print(System.out, roc_data_flip);
  }

  /**
   * Write the test result as a java object to file
   * @param file
   */
  public void writeToFile(String file) throws java.io.IOException {
    DOU_Ser.output(file, this);
  }

  /**
   * read the test result  from a file
   *
   * @param file
   * @return ImgDataCenter
   */
  public static StatTestRst createFromFile(String file) throws Exception {
    return (StatTestRst) DIU_Ser.input(file);
  }

  public static class ClassificationAnalysis {
    public final static String USAGE = "Usage: [-d img_data_center_file] [-r rst_file] [-h html_file_prefix [-i img_path] [-p rst_path]]";

    public static void main(String[] args) {
      try {
        // init configuration
        Configuration.init(args);

        // get the datacenter
        String data_center_name = ReadArgu.getOption('d', args);
        if (data_center_name == null) {
          System.err.println(USAGE);
          System.exit(1);
        }

        // init the datacenter
        String rep_path = Configuration.getParameter("REPRESENTATION_PATH", "");
        ImgDataCenter data_center = ImgDataCenter.createFromFile(
           rep_path + data_center_name);

       String rst_files = ReadArgu.getOption('r', args);
       if (rst_files == null) {
         System.out.println(USAGE);
         System.exit(1);
       }
       String learn_path = Configuration.getParameter("LEARNING_PATH", "");
       StatTestRst rst = StatTestRst.createFromFile(learn_path + rst_files);

       int[][] count = rst.classifyAnalysis(data_center); // right positive curves
       OutputResult.print(System.out, count, count.length, count.length);

     }
     catch (Exception e) {
       e.printStackTrace();
     }

   }

  }

  /**
   * use  description file to analysis classification performance
   */

  public static class ClassificationAnalysis2 {
    public final static String USAGE = "Usage: -d img_desc.desc -r rst_file [-h html_file_prefix [-i img_path] [-p rst_path]]";

    public static void main(String[] args) {
      try {
        // init configuration
        Configuration.init(args);

        // get the datacenter
        String data_center_name = ReadArgu.getOption('d', args);
        if (data_center_name == null) {
          System.err.println(USAGE);
          System.exit(1);
        }

        // init the datacenter
        String rep_path = Configuration.getParameter("REPRESENTATION_PATH", "");
        ImgDescManager idm = new ImgDescManager();
        idm.readFromFile(data_center_name);

       String rst_files = ReadArgu.getOption('r', args);
       if (rst_files == null) {
         System.out.println(USAGE);
         System.exit(1);
       }
       String learn_path = Configuration.getParameter("LEARNING_PATH", "");
       StatTestRst rst = StatTestRst.createFromFile(learn_path + rst_files);

       int[][] count = rst.classifyAnalysis(idm); // right positive curves
       OutputResult.print(System.out, count, count.length, count.length);

     }
     catch (Exception e) {
       e.printStackTrace();
     }

   }

  }

  /**
   * use description file to analysis autolabeling performance.  it shows
   * how many real labels are appeared in the top n auto-generated labels
   */

  public static class LabellingAnalysis {
    public final static String USAGE = "Usage: -d img_desc.desc -r rst_file [-n top_n]";

    public static void main(String[] args) {
      try {
        // init configuration
        Configuration.init(args);

        // get the datacenter
        String data_center_name = ReadArgu.getOption('d', args);
        if (data_center_name == null) {
          System.err.println(USAGE);
          System.exit(1);
        }

        // init the datacenter
        String rep_path = Configuration.getParameter("REPRESENTATION_PATH", "");
        ImgDescManager idm = new ImgDescManager();
        idm.readFromFile(data_center_name);

       String rst_files = ReadArgu.getOption('r', args);
       if (rst_files == null) {
         System.out.println(USAGE);
         System.exit(1);
       }
       String learn_path = Configuration.getParameter("LEARNING_PATH", "");
       StatTestRst rst = StatTestRst.createFromFile(learn_path + rst_files);

       String top_n_str = ReadArgu.getOption('n', args, "1");
       int top_n = Integer.parseInt(top_n_str);

       rst.labellingAnalysis(idm, top_n); // labelling analysis
     }
     catch (Exception e) {
       e.printStackTrace();
     }

   }

  }

  public final static String USAGE = "Usage: [-d img_data_center_file] [-r rst_file[;rst_file]* -[w](in_one_window)] [-h html_file_prefix]]";

  public static void main(String[] args) {
    try {
      // init configuration
      Configuration.init(args);

      // get the datacenter
      String data_center_name = ReadArgu.getOption('d', args);
      if (data_center_name == null) {
        System.err.println(USAGE);
        System.exit(1);
      }
      // init the datacenter
      String rep_path = Configuration.getParameter("REPRESENTATION_PATH", "");
      ImgDataCenter data_center = ImgDataCenter.createFromFile(
         rep_path + data_center_name);

     String rst_files = ReadArgu.getOption('r', args);
     if (rst_files == null) {
       System.out.println(USAGE);
       System.exit(1);
     }
     String learn_path = Configuration.getParameter("LEARNING_PATH", "");

//     String[] rst_file_array = rst_files.split(";");
     String[] rst_file_array = new String[] {
         rst_files
     };

     StatTestRst rst = StatTestRst.createFromFile(learn_path + rst_file_array[0]);

     rst.precisionAndRecallAnalysis(data_center);

     HashMap rpcs = rst.rpcAnalysis(data_center); // right positive curves

     DoubleScoreHashMap roc_scores = rst.getROCScores(rpcs);
     System.out.println("roc scores:");
     System.out.println(roc_scores.toStringBykeyOrder());

//     // verification error print out
//     if (rst instanceof StatTestRstWithVerErr) {
//       DoubleScoreHashMap ver_errors = ((StatTestRstWithVerErr)rst).getVerficationErrors();
//       System.out.println("verification errors:");
//       System.out.println(ver_errors.toStringBykeyOrder());
//     }

     // output the html files
     String html_file_prefix = ReadArgu.getOption('h', args);
     if (html_file_prefix != null) {
       String img_path = Configuration.getParameter("IMAGE_PATH", "");
       String rst_path = Configuration.getParameter("FEATURE_PATH", "");

       if (html_file_prefix.compareToIgnoreCase("TRUE") == 0) {
         html_file_prefix = StringUtil.getFileSuffixNameRemoved(rst_file_array[0]);
       }

       rst.outputSortedHtml(data_center, html_file_prefix, img_path, rst_path);
     }

     String in_one_window = ReadArgu.getOption('w', args);
     if (in_one_window != null) {
       // add more results
       for (int i = 1; i < rst_file_array.length; i++) {
         StatTestRst rst1 = StatTestRst.createFromFile(rst_file_array[i]);
         HashMap rpcs1 = rst1.rpcAnalysis(data_center); // right positive curves
         for (Iterator j = rpcs1.keySet().iterator(); j.hasNext(); ) {
           String key = (String) j.next();
           rpcs.put(key + "." + i, rpcs1.get(key));
         }
       }

       // draw the saved result
       StatTestRst.drawROCs(rpcs);
       StatTestRst.outputROC(rpcs);
     } else {
       StatTestRst.drawROCs(rpcs);
       StatTestRst.outputROC(rpcs);
       for (int i = 1; i < rst_file_array.length; i++) {
         rst = StatTestRst.createFromFile(learn_path + rst_file_array[i]);
         rpcs = rst.rpcAnalysis(data_center); // right positive curves
         StatTestRst.drawROCs(rpcs);
       }
     }

    } catch (Exception e) {
      e.printStackTrace();
    }

  }

  public static class ROCAnalysis {
    public final static String USAGE = "Usage: -d img_desc.desc -r rst_file.rst [-h html_file_prefix]]";

    public static void main(String[] args) {
      try {
        // init configuration
        Configuration.init(args);

        // get the datacenter
        String data_center_name = ReadArgu.getOption('d', args);
        if (data_center_name == null) {
          System.err.println(USAGE);
          System.exit(1);
        }
        String rep_path = Configuration.getParameter("REPRESENTATION_PATH", "");
        ImgDescManager idm = new ImgDescManager();
        idm.readFromFile(data_center_name);

        String rst_files = ReadArgu.getOption('r', args);
        if (rst_files == null) {
          System.out.println(USAGE);
          System.exit(1);
        }
        String learn_path = Configuration.getParameter("LEARNING_PATH", "");

//     String[] rst_file_array = rst_files.split(";");
        String[] rst_file_array = new String[] {
            rst_files
        };

        StatTestRst rst = StatTestRst.createFromFile(learn_path +
            rst_file_array[0]);

        rst.precisionAndRecallAnalysis(idm);

        HashMap rpcs = rst.rpcAnalysis(idm); // right positive curves

        DoubleScoreHashMap roc_scores = rst.getROCScores(rpcs);
        System.out.println("roc scores:");
        System.out.println(roc_scores.toStringBykeyOrder());

//     // verification error print out
//     if (rst instanceof StatTestRstWithVerErr) {
//       DoubleScoreHashMap ver_errors = ((StatTestRstWithVerErr)rst).getVerficationErrors();
//       System.out.println("verification errors:");
//       System.out.println(ver_errors.toStringBykeyOrder());
//     }

        // output the html files
        String html_file_prefix = ReadArgu.getOption('h', args);
        if (html_file_prefix != null) {
          String img_path = Configuration.getParameter("IMAGE_PATH", "");
          String rst_path = Configuration.getParameter("FEATURE_PATH", "");

          if (html_file_prefix.compareToIgnoreCase("TRUE") == 0) {
            html_file_prefix = StringUtil.getFileSuffixNameRemoved(
                rst_file_array[0]);
          }

          rst.outputSortedHtml(idm, html_file_prefix, img_path,
                               rst_path);
        }

        String in_one_window = ReadArgu.getOption('w', args);
        if (in_one_window != null) {
          // add more results
          for (int i = 1; i < rst_file_array.length; i++) {
            StatTestRst rst1 = StatTestRst.createFromFile(rst_file_array[i]);
            HashMap rpcs1 = rst1.rpcAnalysis(idm); // right positive curves
            for (Iterator j = rpcs1.keySet().iterator(); j.hasNext(); ) {
              String key = (String) j.next();
              rpcs.put(key + "." + i, rpcs1.get(key));
            }
          }

          // draw the saved result
          StatTestRst.drawROCs(rpcs);
          StatTestRst.outputROC(rpcs);
        }
        else {
          StatTestRst.drawROCs(rpcs);
          StatTestRst.outputROC(rpcs);
          for (int i = 1; i < rst_file_array.length; i++) {
            rst = StatTestRst.createFromFile(learn_path + rst_file_array[i]);
            rpcs = rst.rpcAnalysis(idm); // right positive curves
            StatTestRst.drawROCs(rpcs);
          }
        }

      }
      catch (Exception e) {
        e.printStackTrace();
      }

    }
  }

  /**
   * Extract the results on specified objects
   */
  public static class ExtractOnObjects {

    public final static String USAGE = "Usage: -r rst_file -o object_name[,object_name]*";

    public static void main(String[] args) {
      try {
        // init configuration
        Configuration.init(args);

        String rst_file = ReadArgu.getOption('r', args);
        if (rst_file == null) {
          System.out.println(USAGE);
          System.exit(1);
        }
        String learn_path = Configuration.getParameter("LEARNING_PATH", "");
        StatTestRst rst = StatTestRst.createFromFile(learn_path + rst_file);


        // get the object names
        String obj_names = ReadArgu.getOption('o', args);
        LinkedList obj_list = null;
        if (obj_names != null) {
          String obj_names_wspace = obj_names.replace('_', ' ');
          String[] objects = obj_names_wspace.split(",");
          obj_list = yi.util.data.Conversion.asLinkedList(objects);
        }

        for (Iterator i=rst.keySet().iterator(); i.hasNext();) {
          Object obj = i.next();
          if (!obj_list.contains(obj)) {
            i.remove();
            System.out.println(obj.toString() + " removed");
          } else {
            System.out.println(obj.toString() + " kept");
          }
        }

        String output_name = StringUtil.getFileSuffixNameRemoved(rst_file) + "." + obj_names + ".rst";
        rst.writeToFile(learn_path + output_name);
        System.out.println("extracted results saved to " + learn_path + output_name);
      }
      catch (Exception e) {
        e.printStackTrace();
      }

    }
  }

  /**
   * Merge two test results.  This is useful specially when you want to
   * merge best results from learning on different objects
   */
  public static class MergeStatTestRst {
    public final static String USAGE = "Usage: -r rst_1 -m rst_2 -o merged_rst";

    public static void main(String[] args) {
      try {

        // init configuration
        Configuration.init(args);

        String rst1 = ReadArgu.getOption('r', args);
        String rst2 = ReadArgu.getOption('m', args);
        String output_file = ReadArgu.getOption('o', args);
        if (rst1 == null || rst2 == null || output_file == null) {
          System.err.println(USAGE);
          System.exit(1);
        }

        String path = Configuration.getParameter("LEARNING_PATH", "");

        StatTestRst data_center_1 = StatTestRst.createFromFile(
            path + rst1);

        StatTestRst data_center_2 = StatTestRst.createFromFile(
            path + rst2);

        data_center_1.putAllInDeep(data_center_2);

        data_center_1.writeToFile(path + output_file);
       }
      catch (Exception e) {
        e.printStackTrace();
      }
    }
  }

  public static String getStatTestRstOutputName(String data_center_name,
                                                String stat_model_adapter_name, String obj_names,
                                                boolean add_path, boolean add_suffix) {
    String file = yi.util.data.StringUtil.getFileSuffixNameRemoved(
        data_center_name) + "." +
        yi.util.data.StringUtil.getClassShortName(stat_model_adapter_name) +
        ( (obj_names == null) ? "" : ("." + obj_names));
    if (add_path) {
      String learn_path = Configuration.getParameter("LEARNING_PATH", "");
      file = learn_path + file;
    }
    if (add_suffix) {
      file = file + ".rst";
    }
    return file;
  }

  /**
   * Merge a list of test results.  This is useful specially when you want to
   * merge best results from learning on different objects
   * -o obj.list -d wang.cp+pc+tp.idc -a yi.iu.imgdb.learn.GMMNNAdapter -n 5
   */
  public static class MergeStatTestRstOfObjs {
    public final static String USAGE = "Usage: -o obj_list_file -d idc -a adapter";

    public static void main(String[] args) {
      try {

        // init configuration
        Configuration.init(args);

        String data_center_name = ReadArgu.getOption('d', args);
        String stat_model_adapter_name = ReadArgu.getOption('a', args);
        String obj_list_file = ReadArgu.getOption('o', args);
        if (stat_model_adapter_name == null || stat_model_adapter_name == null || obj_list_file == null) {
          System.err.println(USAGE);
          System.exit(1);
        }

        String[] obj_list = ReadArgu.readStrings(obj_list_file);
        if (obj_list.length < 2) {
          System.err.println("number of objects in the list file must be more than 2");
          System.exit(1);
        }

        String name = getStatTestRstOutputName(data_center_name, stat_model_adapter_name, obj_list[0], true, true);
        StatTestRst rst1 = StatTestRst.createFromFile(name);
        for (int i=1; i<obj_list.length; i++) {
          name = getStatTestRstOutputName(data_center_name, stat_model_adapter_name, obj_list[i], true, true);
          StatTestRst rst2 = StatTestRst.createFromFile(name);
          rst1.putAllInDeep(rst2);
        }
        name = getStatTestRstOutputName(data_center_name,
                                               stat_model_adapter_name,
                                               null, true, true);
        rst1.writeToFile(name);
       }
      catch (Exception e) {
        e.printStackTrace();
      }
    }
  }

  public static class CopyStatTestRst {
    public final static String USAGE = "Usage: -r rst_1 -o merged_rst";

    public static void main(String[] args) {
      try {

        // init configuration
        Configuration.init(args);

        String rst1 = ReadArgu.getOption('r', args);
        String output_file = ReadArgu.getOption('o', args);
        if (rst1 == null || output_file == null) {
          System.err.println(USAGE);
          System.exit(1);
        }

        String path = Configuration.getParameter("LEARNING_PATH", "");

        StatTestRst data_center_1 = StatTestRst.createFromFile(
            path + rst1);

        StatTestRst rst2 = new StatTestRst();

        rst2.putAllInDeep(data_center_1);

        rst2.writeToFile(path + output_file);
       }
      catch (Exception e) {
        e.printStackTrace();
      }
    }
  }

  public static class TestROCScore {
    public static void main(String[] args) {
      try {
        double[][] roc = yi.util.io.ReadArgu.loadMatlab("test.list");
        System.out.println("roc score = " + StatTestRst.getROCScore(roc));
      }catch (Exception e) {
        e.printStackTrace();
      }
    }
  }

  /**
   * List object probabilities in each image.
   */
  public static class ListObjectProb {
    public final static String USAGE = "Usage: -d img_data_center_file -r rst_file -t dis_match_t";

    public static void main(String[] args) {
      try {
        // init configuration
        Configuration.init(args);

        // get the datacenter
        String data_center_name = ReadArgu.getOption('d', args);
        if (data_center_name == null) {
          System.err.println(USAGE);
          System.exit(1);
        }
        // init the datacenter
        String rep_path = Configuration.getParameter("REPRESENTATION_PATH", "");
        ImgDataCenter data_center = ImgDataCenter.createFromFile(
           rep_path + data_center_name);

       String rst_file = ReadArgu.getOption('r', args);
       if (rst_file == null) {
         System.out.println(USAGE);
         System.exit(1);
       }
       String learn_path = Configuration.getParameter("LEARNING_PATH", "");

       StatTestRst rst = StatTestRst.createFromFile(learn_path + rst_file);

       HashMapHashMap img_obj_score = HashMapHashMap.swapKeys(rst);

       // features used
       String img_path = Configuration.getParameter("IMAGE_PATH", "");
       String ft_path = Configuration.getParameter("FEATURE_PATH", "");
       FeatureDescriptor[] fts = FeatureDescriptor.parse(null, data_center.getFeatureList());

       ArrayList sorted_imgs = new ArrayList(img_obj_score.keySet());
       Collections.sort(sorted_imgs);

       NumberFormat nf = NumberFormat.getNumberInstance();
       nf.setMaximumFractionDigits(2);

       String dis_match_t_str = ReadArgu.getOption('t', args, "0");
       int dis_match_t = Integer.parseInt(dis_match_t_str);

       for (Iterator i=sorted_imgs.iterator(); i.hasNext(); ) {
         String img = (String)i.next();
         HashMap scores = (HashMap)img_obj_score.get(img);
         DoubleScoreHashMap score_map = new DoubleScoreHashMap(scores);
         ArrayList sorted_scores = score_map.getObjectsByOrder();
         Collections.reverse(sorted_scores);

         // check whether those with good scores are in the description
         LinkedList descriptions = data_center.getImgDescriptions(img);
         int dis_match = 0;
         for (int j=0; j<descriptions.size(); j++) {
           if (!descriptions.contains(sorted_scores.get(j))) {
               dis_match ++;
           }
         }
         if (dis_match > dis_match_t || descriptions.size() == 0) continue;
         System.err.println(img + " " + dis_match + " / " + descriptions.size());

         // output html
         String img_name = img_path + img;
         String thumb_img_name = ImgDescManager.toThumbImageName(img_name);
         System.out.println("<a href=" + img_name + "><img src=" + thumb_img_name + " ></a><br>");
         for (int j=0; j<sorted_scores.size(); j++) {
           Object obj = sorted_scores.get(j);
           if (descriptions.contains(obj)) {
             System.out.print("<b>" + obj.toString() + "</b>");
           } else {
             System.out.print(obj.toString());
           }
           System.out.print("(" + nf.format(score_map.getScore(obj)*100) + ") ");
         }
         System.out.println("<br>");
         for (int f=0; f<fts.length; f++) {
           FeatureDescriptor ft = fts[f];
           String cdemo = FeatureDescriptor.getFeatureColorDemoName(ft, img, true);
           String gdemo = FeatureDescriptor.getFeatureGraphicDemoName(ft, img, true);
           System.out.println("<img src=" + cdemo + " >");
           System.out.println("<img src=" + gdemo + " >");
         }
         System.out.println("<br>");
       }

      } catch (Exception e) {
        e.printStackTrace();
      }

    }

  }

  /**
   * Replace empty space in url as "_"
   * @param url
   * @return
   */
  String replaceEmptySpace(String url) {
    return url.replaceAll(" " , "_");
  }

  /**
   * Print the index file for web demo
   * @param data_center
   * @param pre_html_file_prefix
   * @param ann_html_file_prefix
   * @param img_path
   * @param rst_path
   */
  public void printIndex4WebPage(ImgDataCenter data_center,
                                 String pre_html_file_prefix,
                                 String ann_html_file_prefix,
                                 String img_path, String rst_path) {
    int WEB_IMG_PER_ROW = 5;

    String out_name = ann_html_file_prefix + "." + "index" + ".html";
    PrintStream out = OutputResult.getPrintStream(out_name);

    out.println("<html><body>");
    out.println("<h3>Browse Computer Prediction: " + "</h3>");
    out.println("<table>");

    // analysis each object
    List objects = new ArrayList(keySet());
    Collections.sort(objects);
    int index = 0;
    for (Iterator i = objects.iterator(); i.hasNext(); index++) {
      Object the_obj = i.next();
      String pre_name = pre_html_file_prefix + "." + the_obj + ".html";
      pre_name = replaceEmptySpace(pre_name);

      if (index % WEB_IMG_PER_ROW == 0) {
        if (index != 0) {
          out.println("</tr>");
        }
        out.println("<tr>");
      }

      out.print("<td>");
      out.print("<a href=" + pre_name + ">" + the_obj + "</a>");
      out.println("</td>");
    }
    out.println("</tr></table>");

    out.println("<h3>Browse Manual Annotation: " + "</h3>");
    out.println("<table>");

    index = 0;
    for (Iterator i = objects.iterator(); i.hasNext(); index++) {
      Object the_obj = i.next();
      String ann_name = ann_html_file_prefix + "." + the_obj + ".html";
      ann_name = replaceEmptySpace(ann_name);

      if (index % WEB_IMG_PER_ROW == 0) {
        if (index != 0) {
          out.println("</tr>");
        }
        out.println("<tr>");
      }

      out.print("<td>");
      out.print("<a href=" + ann_name + ">" + the_obj + "</a>");
      out.println("</td>");
    }
    out.println("</tr></table></body></html>");
    out.close();
  }

  /**
   * Print computer prediction pages for web demo
   * @param data_center
   * @param pre_html_file_prefix
   * @param ann_html_file_prefix
   * @param img_path
   * @param rst_path
   */
  public void printPredictionWebPages(ImgDataCenter data_center,
                                String pre_html_file_prefix, String ann_html_file_prefix,
                               String img_path, String rst_path) {
   int WEB_OBJ_QUERY_RETURNS = 12;
   int WEB_IMG_PER_ROW = 3;

   HashMapHashMap img_obj_score = HashMapHashMap.swapKeys(this);

   // analysis each object
   List objects = new ArrayList(keySet());
   Collections.sort(objects);
   for (Iterator i = objects.iterator(); i.hasNext(); ) {
     Object the_obj = i.next();
     String out_name = pre_html_file_prefix + "." + the_obj + ".html";
     out_name = replaceEmptySpace(out_name);
     PrintStream out = OutputResult.getPrintStream(out_name);

     out.println("<STYLE TYPE=\"text/css\">");
     out.println("                      <!--");
     out.println("                      A {");
     out.println("                          text-decoration:none;");
     out.println("                      }");
     out.println("                      -->");
     out.println("                      </STYLE>");

     out.println("<html><body>");
     out.println("<h3>Sorted by Prediction: " + the_obj.toString() + "</h3>");
     out.println("<table>");

     DoubleScoreHashMap one_obj_score = new DoubleScoreHashMap(
         (HashMap) get(the_obj));
     // sort the images according to their score
     ArrayList the_images = new ArrayList(one_obj_score.keySet());
     Collections.sort(the_images, one_obj_score);
     // sort from more positive to less positive
     if (m_higherIsBetter) {
       Collections.reverse(the_images);
     }

     int index = 0;
     for (Iterator j = the_images.iterator(); j.hasNext(); index++) {
       if (index >= WEB_OBJ_QUERY_RETURNS)
         break;

       if (index % WEB_IMG_PER_ROW == 0) {
         if (index != 0) {
           out.println("</tr>");
         }
         out.println("<tr>");
       }

       String the_img = (String) j.next();

       String the_img_thumb = yi.iu.imgdb.desc.ImgDescManager.toThumbImageName(
           the_img);
       out.print("<td>");
       printImgEntry4WebPage(out, the_img, data_center, img_obj_score, the_obj,
                             img_path, pre_html_file_prefix, ann_html_file_prefix);
       out.println("</td>");
     }
     out.println("</tr></table></body></html>");
     out.close();
    }
  }

  /**
   * Print manual annotation pages for web demo
   * @param data_center
   * @param pre_html_file_prefix
   * @param ann_html_file_prefix
   * @param img_path
   * @param rst_path
   */
  void printAnnotationWebPages(ImgDataCenter data_center,
                            String pre_html_file_prefix, String ann_html_file_prefix,
                            String img_path, String rst_path) {

    int WEB_IMG_PER_ROW = 3;
    HashMapHashMap img_obj_score = HashMapHashMap.swapKeys(this);

    Collection keyword_list = data_center.getObjectList();

    ArrayList files = new ArrayList(data_center.getImageList());
    Collections.sort(files);

    for (Iterator k=keyword_list.iterator(); k.hasNext(); ) {
      String the_obj = (String)k.next();

      String out_name = ann_html_file_prefix + "." + the_obj + ".html";
      out_name = replaceEmptySpace(out_name);
      PrintStream out = OutputResult.getPrintStream(out_name);

      out.println("<STYLE TYPE=\"text/css\">");
      out.println("                      <!--");
      out.println("                      A {");
      out.println("                          text-decoration:none;");
      out.println("                      }");
      out.println("                      -->");
      out.println("                      </STYLE>");

      out.println("<html><body>");
      out.println("<h3>Browse Annotation: " + the_obj.toString() + "</h3>");
      out.println("<table>");

      // only show those image containing the specified keywords
      int index = 0;
      for (Iterator i = files.iterator(); i.hasNext(); ) {
        String the_img = (String) i.next();
        LinkedList desc = data_center.getImgDescriptions(the_img);
        if (!desc.contains(the_obj)) {
          continue;
        }

        if (index % WEB_IMG_PER_ROW == 0) {
          if (index != 0) {
            out.println("</tr>");
          }
          out.println("<tr>");
        }
        index++;

        String the_img_thumb = yi.iu.imgdb.desc.ImgDescManager.toThumbImageName(
            the_img);
        out.print("<td>");
        printImgEntry4WebPage(out, the_img, data_center, img_obj_score, the_obj,
                              img_path, pre_html_file_prefix, ann_html_file_prefix);
        out.println("</td>");
      }
      out.println("</tr></table></body></html>");
      out.close();
    }

  }

  /**
   * Print a image entry for web pages
   * @param out
   * @param img
   * @param data_center
   * @param img_obj_score
   * @param the_obj
   * @param img_path
   * @param pre_html_file_prefix
   * @param ann_html_file_prefix
   */
  void printImgEntry4WebPage(PrintStream out, String img,
                             ImgDataCenter data_center, HashMapHashMap img_obj_score,
                             Object the_obj, String img_path,
                             String pre_html_file_prefix, String ann_html_file_prefix
                             ) {

     NumberFormat nf = NumberFormat.getNumberInstance();
     nf.setMaximumFractionDigits(1);

     HashMap scores = (HashMap) img_obj_score.get(img);
     DoubleScoreHashMap score_map = new DoubleScoreHashMap(scores);
     ArrayList sorted_scores = score_map.getObjectsByOrder();
     Collections.reverse(sorted_scores);

     LinkedList descriptions = data_center.getImgDescriptions(img);

     String the_img_thumb = yi.iu.imgdb.desc.ImgDescManager.toThumbImageName(img);

     out.println("<a href=" + img_path + img + "><img src=" + img_path + the_img_thumb +
                        " ALT=" + img_path + the_img_thumb + "></a><br>");
     out.println("<B>Computer Prediction:</B>" + "<br>");
     out.print("<font size=\"-1\" face=\"Verdana, Arial, Helvetica, Sans-Serif\">");
     for (int j = 0; j < sorted_scores.size(); j++) {
       if (j!=0) out.print(", ");
       Object obj = sorted_scores.get(j);
       if (score_map.getScore(obj) < 0.5) break;
//       if (descriptions.contains(obj)) out.print("<b>");
       String pre_html_name = pre_html_file_prefix + "." + obj + ".html";
       pre_html_name = replaceEmptySpace(pre_html_name);
       out.print("<a href=" + pre_html_name + ">");
       if (obj.toString().compareToIgnoreCase(the_obj.toString())==0) out.print("<font color=#FF0000>");
       else out.print("<font color=#0000FF>");
       out.print(obj);
       out.print("</font>");
       out.print("</a>");
//       if (descriptions.contains(obj)) out.print("</b>");
       out.print("(" + nf.format(score_map.getScore(obj) * 100) + ") ");
     }
     out.print("</font>");
     out.println("<br>");
     out.println("<b>Manual Annotation:</b>" + "<br>");
     out.print("<font size=\"-1\" face=\"Verdana, Arial, Helvetica, Sans-Serif\">");
     Collections.sort(descriptions);
     int i_index = 0;
     for (Iterator i=descriptions.iterator(); i.hasNext(); i_index++) {
       if (i_index!=0) out.print(", ");
       Object obj = i.next();
       String ann_html_name = ann_html_file_prefix + "." + obj + ".html";
       ann_html_name = replaceEmptySpace(ann_html_name);
       out.print("<a href=" + ann_html_name + ">");
       if (obj.toString().compareToIgnoreCase(the_obj.toString())==0) out.print("<font color=#FF0000>");
       else out.print("<font color=#0000FF>");
       out.print(obj);
       out.print("</font>");
       out.print("</a>");
     }
     out.print("</font>");
  }

  public static class GenWebDemo {
    public final static String USAGE = "Usage: -d img_data_center_file -r rst_file";

    public static void main(String[] args) {
      try {
        // init configuration
        Configuration.init(args);

        String img_path = Configuration.getParameter("IMAGE_PATH", "");
        String rst_path = Configuration.getParameter("FEATURE_PATH", "");
        String rep_path = Configuration.getParameter("REPRESENTATION_PATH", "");
        String learn_path = Configuration.getParameter("LEARNING_PATH", "");

        // get the datacenter
        String data_center_name = ReadArgu.getOption('d', args);
        if (data_center_name == null) {
          System.err.println(USAGE);
          System.exit(1);
        }
        // init the datacenter
        ImgDataCenter data_center = ImgDataCenter.createFromFile(
            rep_path + data_center_name);

        String rst_files = ReadArgu.getOption('r', args);
        if (rst_files == null) {
          System.out.println(USAGE);
          System.exit(1);
        }

        StatTestRst rst = StatTestRst.createFromFile(learn_path + rst_files);

        // output the html files
        String pre_html_file_prefix = StringUtil.getFileSuffixNameRemoved(rst_files);
        String ann_html_file_prefix = StringUtil.getFileSuffixNameRemoved(data_center_name);

        rst.printPredictionWebPages(data_center, pre_html_file_prefix, ann_html_file_prefix, img_path, rst_path);
        rst.printAnnotationWebPages(data_center, pre_html_file_prefix, ann_html_file_prefix, img_path, rst_path);
        rst.printIndex4WebPage(data_center, pre_html_file_prefix, ann_html_file_prefix, img_path, rst_path);
      } catch (Exception e) {
        e.printStackTrace();
      }

    }
  }

}


class PrecisionAndRecall {
  double precision;
  double recall;
  public PrecisionAndRecall(double precision, double recall) {
    this.precision = precision;
    this.recall = recall;
  }
}