//Source file: D:\\Projects\\src\\yi\\iu\\imgdb\\learn\\ObjGMMAdapter.java

//Source file: D:\\Projects\\src\\ObjGMMAdapter.java

package yi.iu.imgdb.learn;

import java.util.LinkedList;
import java.util.*;
import yi.util.io.*;
import yi.stat.em.*;
import org.jmat.data.*;
import org.jmat.gui.*;
import yi.ds.*;
import yi.util.io.output.*;
import yi.util.config.*;
import yi.ds.hash.*;

/**
 * Modelize each object as a Gaussian Mixture Model.
 */
public class ObjGMMAdapter
    extends StatModelAdapter implements FeatureScoringCapable{

  /**
   * HashMap of GaussianMixtureModel.  Each entry corresponds to one object
   */
  HashMap m_models = new HashMap();

  /**
   * HashMap of GaussianMixtureModel.  Each entry corresponds to the negative
   * model of one object
   */
  HashMap m_negativeModels = new HashMap();

  /**
   * This adapter only learns on one feature
   */
  String m_feature = null;

  /** @todo since no verifying required now, lower the portion  */
  /**
   * How much portion of learning data will be used to verify the  model
   */
  public double VERIFY_PORTION = 0;

  /**
   * When draw the distribution, how many of points will be sampled.
   */
  public int DISTRIBUTION_DRAW_SLICE_NUM = 100;

  /**
   * Whether to show the compaison of data histogram and model distribution
   * one plot
   */
  public boolean SHOW_DATA_MODEL_PLOT = true;

  /**
   * @param data_center
   * @param features
   * @param imgs
   */
  public ObjGMMAdapter(ImgDataCenter data_center, LinkedList img_list, LinkedList ft_list, LinkedList obj_list)
  {
    super(data_center, img_list, ft_list, obj_list);

    // this method only learns on one feature
    m_feature = (String)m_featureList.getFirst();

    /** @todo regenerate image data center */
    int feature_len = data_center.getFeatureLength(m_feature);
//    int feature_len = m_imgDataCenter.getFeatureLength(m_feature);

    // learn the objects one by one
    for (Iterator o = m_objectList.iterator(); o.hasNext(); ) {
      Object object = o.next();

      System.out.println();
      System.out.println("learning on \"" + object + "\"" + "\n");

      // seperate training set and verifying set
      LinkedList posTrainImgs = new LinkedList();
      LinkedList negTrainImgs = new LinkedList();
      LinkedList posVerifyImgs = new LinkedList();
      LinkedList negVerifyImgs = new LinkedList();
      seperateVerifyImgsFromTrainImgs(img_list, object, VERIFY_PORTION,
                                      posTrainImgs, negTrainImgs,
                                      posVerifyImgs, negVerifyImgs);


      // group the feature vectors in the negative ones to construct negative model
      double[][] neg_data = m_imgDataCenter.groupFeatureVectors(negTrainImgs, m_feature);
      System.out.println("total negative training feature vectors #= " +
                         neg_data.length + "\n");
      /** @todo output data for analysis */
      //OutputResult.outputResult("neg.dat", neg_data);

      // normalize the data */
      /** @todo change to feature length */
      double[] max_neg_data = yi.stat.StatUtil.maximumOnCol(neg_data);
      double[] min_neg_data = yi.stat.StatUtil.minimumOnCol(neg_data);

      // get the negative model first
      int M_IN_BG = 20;
      System.out.println("finding negative model... m = " + M_IN_BG);
//      GaussianMixtureModel neg_gmm = new GaussianMixtureModel(M_IN_BG, neg_data, null, null); // suppose the data is within [0,1)
      GaussianMixtureModel neg_gmm = new AlignedGaussianMixtureModel(M_IN_BG, neg_data, (double[])null); // suppose the data is within [0,1)
      neg_gmm.learnTheModel();

      m_negativeModels.put(object, neg_gmm);

      // group the feature vectors in the positive ones to construct positive model
      double[][] data = m_imgDataCenter.groupFeatureVectors(posTrainImgs, m_feature);
      System.out.println("total positive training feature vectors #= " +
                         data.length + "\n");

      // maximum of the training positives.
      double[] max_data = yi.stat.StatUtil.maximumOnCol(data);
      double[] min_data = yi.stat.StatUtil.minimumOnCol(data);

      double maximum = Math.max(yi.stat.StatUtil.maximum(max_data),yi.stat.StatUtil.maximum(max_neg_data));
      double minimum = Math.max(yi.stat.StatUtil.minimum(min_data),yi.stat.StatUtil.minimum(min_neg_data));



//      OutputResult.print(System.out, data);
//      Matrix matrix = new Matrix(data);
//      Plot3DPanel pp = new Plot3DPanel();
//      matrix.toPlot3DPanel(pp, "hsb", "SCATTER");
//      new ExistablePlotViewer("ROC", pp);

      /** @todo change the feature length to show plot */
      // for SHOW_DISTRIBUTION, save the frequency functions
      // each entry is for one dimension.  Each entry of the linked list
      // is the one for a specified m (as in m mixture models)
      LinkedList[] ffs = new LinkedList[feature_len];

      /** @todo see whether m can be fixed; or a big m is enough */
      int M_IN_POS = 20;
      GaussianMixtureModel best_gmm = learnObjectModel(
          M_IN_POS, M_IN_POS, data, min_data, max_data, minimum, maximum,
          neg_gmm, posVerifyImgs, negVerifyImgs, ffs);


      m_models.put(object, best_gmm);

//      // output the distributions
//      for (int i=0; i<ffs.length; i++) {
//        double[][] ff_w_m = yi.util.data.Conversion.to2DDoubleArray(ffs[i]);
//        OutputResult.outputResult("distributions", ff_w_m);
//      }


      // draw the distributions fro each dimension of the feature
      if (SHOW_DATA_MODEL_PLOT) {
        System.out.println("object model: " + best_gmm);
        for (int i = 0; i < ffs.length; i++) {
          Plot2DPanel pp = best_gmm.showDataModelOnPanel(ffs[i], i, minimum, maximum,
              DISTRIBUTION_DRAW_SLICE_NUM);
          new FrameView("obj_D" + i , pp);
        }
        System.out.println("background model: " + neg_gmm);
        LinkedList[] neg_ffs = new LinkedList[feature_len];
        try {
          calculateDistributions(neg_gmm, neg_ffs, minimum, maximum,
                                 DISTRIBUTION_DRAW_SLICE_NUM);
        }catch (Exception e) {
          e.printStackTrace();
        }
        for (int i = 0; i < neg_ffs.length; i++) {
          Plot2DPanel pp = neg_gmm.showDataModelOnPanel(neg_ffs[i], i, minimum, maximum,
              DISTRIBUTION_DRAW_SLICE_NUM);
          new FrameView("bg_D" + i, pp);
        }
      }
    } // end loop of objects
  }


  /**
   * add the distribution (each one is 1D) of current model to the passed in
   * linked list
   */
  void calculateDistributions(
      GaussianMixtureModel gmm, LinkedList[] ffs,
      double min, double max, int slice_num) throws GaussianModelSingularException{
    for (int i=0; i<ffs.length; i++) {
      double[] ff = gmm.getOneDFrequencyFunction(i, min, max, slice_num);
      if (ffs[i] == null) ffs[i] = new LinkedList();
      ffs[i].add(ff);
    }
  }

  /**
   * Learn the object model.  This method increases the number of m if doing
   * so improve the performance
   * @param min_m
   * @param max_m
   * @param data
   * @param max_data
   * @param neg_gmm
   * @param posVerifyImgs
   * @param negVerifyImgs
   * @param ffs
   */
  GaussianMixtureModel learnObjectModel(int min_m, int max_m, double[][] data, double[] min_data, double[] max_data,
                                        double minimum, double maximum,
                        GaussianMixtureModel neg_gmm,
                        LinkedList posVerifyImgs, LinkedList negVerifyImgs,
                        LinkedList[] ffs) {
    double last_score = -Double.MAX_VALUE;
    GaussianMixtureModel best_gmm = null;

    // increase kernels
    for (int m = min_m; m<=max_m; m=Math.max(m+1, (int)(m*1.5))) {

      System.out.println("kernel # = " + m + "\n");

      // training
      /** @todo compare aligned and general Gaussian */
//        GaussianMixtureModel gmm = new GaussianMixtureModel(m, data, null, null); // suppose the data is within [0,1)
      GaussianMixtureModel gmm = new AlignedGaussianMixtureModel(m, data, (double[])null); // suppose the data is within [0,1)
      gmm.learnTheModel();
//        System.out.println(gmm);

      /** @todo compare score() and score2().  score2() has more strong theoratic
       * background */
      // verify on positive cases
      double total_score = score2(gmm, neg_gmm, posVerifyImgs, negVerifyImgs);

      System.out.println("total score = " + total_score);

      // get the distribution of current model
      try {
        /** @todo the min and max of plots */
        calculateDistributions(gmm, ffs, minimum, maximum, DISTRIBUTION_DRAW_SLICE_NUM);
      } catch (Exception e) {
        e.printStackTrace();
      }

      // compare the current score to the last one.  If not better, just quit
      if (total_score > last_score) {
        last_score = total_score;
        best_gmm = gmm;
      }
      else {
        break;
      }
    } // end of loop of m (mixture model number)

    return best_gmm;

  }




  /**
   * Given score for each images in the given image list for each object.
   * The score here is after log.
   * @param imgs
   * @return StatTestRst
   */
  public StatTestRst test(java.util.LinkedList imgs) {
    StatTestRst rst = new StatTestRst();
    for (Iterator o = m_objectList.iterator(); o.hasNext(); ) {
      Object object = o.next();
      System.out.println(object);
      GaussianMixtureModel gmm = (GaussianMixtureModel)m_models.get(object);
      GaussianMixtureModel neg_gmm = (GaussianMixtureModel)m_negativeModels.get(object);
      for (Iterator j = imgs.iterator(); j.hasNext(); ) {
        String img = (String)j.next();
        double score = score(gmm, neg_gmm, img);
        rst.putScore(object, img, score);
        System.out.println(score);
      }
    }
    return rst;
  }

  /**
   * Give a score of current positive and negative models.  How those models
   * fit the positive and negative instances
   * @param gmm
   * @param neg_gmm
   * @param posVerifyImgs
   * @param negVerifyImgs
   * @return
   */
  double score(GaussianMixtureModel gmm, GaussianMixtureModel neg_gmm,
               LinkedList posVerifyImgs, LinkedList negVerifyImgs) {
    double total_score = 0;
    for (Iterator j = posVerifyImgs.iterator(); j.hasNext(); ) {
      String img = (String) j.next();
      try {
        double score = score(gmm, neg_gmm, img);
//            System.out.println("+" + img + " = " + score);
        total_score += score;
      }
      catch (NoSuchElementException e) {
        System.out.println("there is no feature available for " + img +
                           ", on feature " + m_feature);
      }
    }

// verify on negative cases
    for (Iterator j = negVerifyImgs.iterator(); j.hasNext(); ) {
      String img = (String) j.next();
      try {
        double score = score(gmm, neg_gmm, img);
//            System.out.println("-" + img + " = " + score);
        total_score -= score;
      }
      catch (NoSuchElementException e) {
        System.out.println("there is no feature available for " + img +
                           ", on feature " + m_feature);
      }
    }
    return total_score;

  }

  /**
   * Give a score of current positive and negative models.  How those models
   * fit the positive and negative instances
   * @param gmm
   * @param neg_gmm
   * @param posVerifyImgs
   * @param negVerifyImgs
   * @return
   */
  double score2(GaussianMixtureModel gmm, GaussianMixtureModel neg_gmm,
                LinkedList posVerifyImgs, LinkedList negVerifyImgs) {
    double total_score = 0;
    double[][] pos = m_imgDataCenter.groupFeatureVectors(posVerifyImgs, m_feature);
    double[][] neg = m_imgDataCenter.groupFeatureVectors(negVerifyImgs, m_feature);
    int pos_num = pos.length;
    int neg_num = neg.length;

    double joint_prob = 0;

    try {

      for (int i = 0; i < pos.length; i++) {
        double[] f = pos[i];
        double dp = gmm.getDensity(f);
        double dn = neg_gmm.getDensity(f);
        double p = dp * pos_num / (dp * pos_num + dn * neg_num);
        joint_prob += Math.log(p);
      }

      for (int i = 0; i < neg.length; i++) {
        double[] f = neg[i];
        double dp = gmm.getDensity(f);
        double dn = neg_gmm.getDensity(f);
        double p = dn * neg_num / (dp * pos_num + dn * neg_num);
        joint_prob += Math.log(p);
      }
    } catch (ModelSingularException e) {
      e.printStackTrace();
    }

    return joint_prob;

  }


  /**
   * p({f}|obj)/p({f}|bg} after log
   * @param gmm
   * @param neg_gmm
   * @param img
   * @return
   */
  double score(GaussianMixtureModel gmm, GaussianMixtureModel neg_gmm, String img) {
    // get the feature vectors
    FeatureRepresentationList features = m_imgDataCenter.getImgFeatureList(
        img, // image name
        m_feature);
    double[][] data = features.as2DDoubleArray();
    double pos_score = jointDensity(gmm, data);
    double neg_score = jointDensity(neg_gmm, data);
    return pos_score - neg_score;
  }

  /**
   * Calculate the joint density of independent feature data:
   * p(d1,d2,d2...) = p(d1)*p(d2)*p(d3)*...
   * The score is the one after ln()
   * @param gmm
   * @param features
   * @return
   */
  public double jointDensity(GaussianMixtureModel gmm, double[][] data) {

    double density = 0;
    for (int i = 0; i < data.length; i++) {
      double[] f = data[i];
      double d = 0;
      try {
        d = gmm.getDensity(f);
      } catch (ModelSingularException e) {
        e.printStackTrace();
      }
      density += Math.log(d);
      System.out.print(Double.toString(d) + ": " );
      OutputResult.print(System.out, f);
    }
    return density;
  }

  /**
   * Given a feature vector, return its score of the given object
   * @param object
   * @param feature
   * @return
   */
  public double getFeatureScore(double[] feature, Object object) {
    GaussianMixtureModel gmm = (GaussianMixtureModel)m_models.get(object);
    GaussianMixtureModel neg_gmm = (GaussianMixtureModel)m_negativeModels.get(object);

    try {
      double d1 = gmm.getDensity(feature); // p(feature|object)
      double d2 = neg_gmm.getDensity(feature); // p(feature|non-object)
      return Math.log(d1) - Math.log(d2);
    } catch (ModelSingularException e) {
      e.printStackTrace();
      return 0;
    }
  }


  /**
   * Given the learning images, seperate them to four parts: positive training
   * images, negative training images, positive verifying images, and negative
   * verifying images.  This code ensures that the portion of positive images
   * in training images is similar to that in verifying images
   * @param img_list
   * @param object
   * @param VERIFY_PORTION
   * @param posTrainImgs
   * @param negTrainImgs
   * @param posVerifyImgs
   * @param negVerifyImgs
   */
  void seperateVerifyImgsFromTrainImgs(LinkedList img_list, Object object,
                                       double verify_portion,
                                       LinkedList posTrainImgs,
                                       LinkedList negTrainImgs,
                                       LinkedList posVerifyImgs,
                                       LinkedList negVerifyImgs) {
    // seperate training set and verifying set
    // first seperate positive and negative set to make their portion in
    // training set is similar to those in verifying set
    LinkedList negative = new LinkedList();
    LinkedList positive = new LinkedList();
    for (Iterator i = img_list.iterator(); i.hasNext(); ) {
      String img = (String) i.next();
      LinkedList desc_list = m_imgDataCenter.getImgDescriptions(img);
      if (desc_list !=null && desc_list.contains(object)) {
        positive.add(img);
      }
      else {
        negative.add(img);
      }
    }

    // seperate part of the training instance for verification
    int pos_verify_n = (int) (positive.size() * verify_portion);
    int j_index = 0;
    for (Iterator j = positive.iterator(); j.hasNext(); j_index++) {
      if (j_index < pos_verify_n) {
        posVerifyImgs.add(j.next());
      }
      else {
        posTrainImgs.add(j.next());
      }
    }
    int neg_verify_n = (int) (negative.size() * verify_portion);
    j_index = 0;
    for (Iterator j = negative.iterator(); j.hasNext(); j_index++) {
      if (j_index < neg_verify_n) {
        negVerifyImgs.add(j.next());
      }
      else {
        negTrainImgs.add(j.next());
      }
    }

    System.out.println("Positive Training set: " + posTrainImgs.size());
//    OutputResult.print(System.out, posTrainImgs);
    System.out.println("Negative Training set: " + negTrainImgs.size());
//    OutputResult.print(System.out, negTrainImgs);
    System.out.println("Positive Verifying set: " + posVerifyImgs.size());
//    OutputResult.print(System.out, posVerifyImgs);
    System.out.println("Negative Verifying set: " + negVerifyImgs.size());
//    OutputResult.print(System.out, negVerifyImgs);
  }

  public static class DemoScore {

    public final static String USAGE = "Usage: [-d img_data_center_file] [-m model_file] [-i img_list_file] [-f feature]";

    public static void main(String[] args) {

      // init configuration
      Configuration.init(args);

      try {

        // read in an ObjGMMAdapter object
        String model_file = ReadArgu.getOption(
            'm', args,
            "model0.ser"
            );

        ObjGMMAdapter gmm = (ObjGMMAdapter)ObjGMMAdapter.createFromFile(model_file);

        // init the datacenter
        String data_center_name = ReadArgu.getOption(
            'd', args,
            "groundtruth.rgn.ser"
            );

        ImgDataCenter data_center = ImgDataCenter.createFromFile(
            data_center_name);

        // get the image list file name
        String img_list_file = ReadArgu.getOption('i', args);

        // image list
        LinkedList img_list = null;
        if (img_list_file != null) {
          String[] images = yi.util.io.ReadArgu.readStrings(img_list_file);
          img_list = yi.util.data.Conversion.asLinkedList(images);
        }

        /** @todo custom the feature name */
        String feature = ReadArgu.getOption('f', args, "yi.iu.processor.texture.reg.DS_PolygonRegionTextureStat");

        HashMap rst_file_suffix_map = new HashMap();
        rst_file_suffix_map.put("yi.iu.processor.texture.reg.DS_PolygonRegionTextureStat", ".s256.text.reg.gif");
        rst_file_suffix_map.put("yi.iu.processor.color.reg.DS_PolygonRegionColorStat", ".s256.color.reg.gif");
        rst_file_suffix_map.put("yi.iu.processor.clc.LineClusterSets", ".CLC_LineStruct.bound.gif");

        String obj = ReadArgu.getOption(
            'd', args,
            "tree"
            );


        // loop through the image and their features
        for (Iterator i=img_list.iterator(); i.hasNext();) {
          String img = (String)i.next();
          PolygonFeatureRepresentationList rfrl =
              (PolygonFeatureRepresentationList)data_center.getImgFeatureList(img, feature);
          int[] score_img = rfrl.drawScoreDemo(obj, gmm, 128, 10);
          for (int j=0; j<score_img.length; j++) {
            if (score_img[j] < 0) score_img[j] = 0;
            if (score_img[j] > 255) score_img[j] = 255;
          }
          int[] pixels_img = yi.util.ImageUtil.makeNewImageFromGray(score_img);
          yi.util.io.OutputResult.outputGIFImage(img + ".score.gif", pixels_img,
                                                 rfrl.imgWidth, rfrl.imgHeight);
          System.out.println("<P>" +
                             "<IMG SRC=\"" + img + "\" width=256>" +
                             "<IMG SRC=\"" + img + (String)rst_file_suffix_map.get(feature) + "\" width=256>" +
                             "<IMG SRC=\"" + img + ".score.gif\" width=256>" +
                             "</P>");
        }

      } catch (Exception e) {
        e.printStackTrace();
      }
    }
  }

  public static class Test {

    public final static String USAGE = "Usage: [-d img_data_center_file] [-m model_file] [-i img_list_file] [-f feature] [-r rst_file_name]";

    public static void main(String[] args) {

      // init configuration
      Configuration.init(args);

      try {
        // read in an ObjGMMAdapter object
        String model_file = ReadArgu.getOption(
            'm', args,
            "model0.ser"
            );

        ObjGMMAdapter gmm = (ObjGMMAdapter)ObjGMMAdapter.createFromFile(model_file);

        // init the datacenter
        String data_center_name = ReadArgu.getOption(
            'd', args,
            "groundtruth.ser"
            );

        ImgDataCenter data_center = ImgDataCenter.createFromFile(
            data_center_name);

        // get the image list file name
        String img_list_file = ReadArgu.getOption('i', args);

        // image list
        LinkedList img_list = null;
        if (img_list_file != null) {
          String[] images = yi.util.io.ReadArgu.readStrings(img_list_file);
          img_list = yi.util.data.Conversion.asLinkedList(images);
        }

        // System.out.println(gmm.m_models);

        StatTestRst rst = gmm.test(img_list);

        HashMap rpcs = rst.rpcAnalysis(data_center); // right positive curves


        DoubleScoreHashMap roc_scores = rst.getROCScores(rpcs);
        System.out.println("roc scores:");
        System.out.println(roc_scores.toStringBykeyOrder());

        Plot2DPanel pp = rst.drawROCs(rpcs);

        String rst_output_file = ReadArgu.getOption('r', args);
        if (rst_output_file != null) {
          pp.setFixedBounds(0,0,1);
          pp.setFixedBounds(1,0,1);
          pp.invalidate();
          pp.toGraphicFile(new java.io.File(rst_output_file + ".png"));
        }

      } catch (Exception e) {
        e.printStackTrace();
      }
    }
  }

}