package yi.iu.imgdb.learn;

import java.util.*;
import yi.iu.imgdb.learn.absrgn.*;
import yi.stat.em.*;
import yi.stat.nn.*;
import yi.util.config.*;
import yi.util.io.*;
import yi.ds.hash.*;
import org.jmat.gui.*;
import java.io.*;
import java.awt.Dimension;
import yi.iu.imgdb.learn.absrgn.AbsRgnFeatureGenerator;
import yi.iu.imgdb.learn.absrgn.AbsRgnBiRelFeatures;

/**
 * GMMNNWithBiRelAdapter is GMM-NN adapter with the ability to adding in
 * bi-relationships
 */

public class GMMNNWithBiRelAdapter extends GMMNNAdapter{

  static final long serialVersionUID = -665398539544650123L;
  static String BIRELFEATURE_NAME = "BiRel";

  public GMMNNWithBiRelAdapter(ImgDataCenter data_center, LinkedList img_list, LinkedList ft_list, LinkedList obj_list)
  {
      super(data_center, img_list, ft_list, obj_list);
  }

  /**
   * Construct abstract region features
   * @param object
   * @param img_list
   */
  public void constructAbstractRegionModel(String object, LinkedList img_list) {

      constructAbstractRegionFeatures(object, img_list);

      // learn an additional gmm model for bi-relation
      // seperate training set and verifying set
      LinkedList posTrainImgs = new LinkedList();
      LinkedList negTrainImgs = new LinkedList();
      seperatePosImgsFromNegImgs(img_list, object, posTrainImgs, negTrainImgs);

      // learn a gmm for "BiRel"
      try {
//      GMM_CONSTR_RETRIES = 1;
//      GMM_BREAK_THRESHOLD = 1e-3;
        int m = M;
        while (true) {
          learnGaussianMixtureModelOnFeature(object, BIRELFEATURE_NAME,
                                             posTrainImgs, m);

          // check the model
          GaussianMixtureModel gmm = (GaussianMixtureModel) m_gmmModels.get(
              object, BIRELFEATURE_NAME);
          HashMap means = gmm.getComponentMeans();
          HashMap covs = gmm.getComponentCovariance();
          ArrayList keys = new ArrayList(means.keySet());
          Collections.sort(keys);
          System.out.println(BIRELFEATURE_NAME + " Gaussian mixture model:");
          for (Iterator i = keys.iterator(); i.hasNext(); ) {
            Object key = i.next();
            double[] mean = (double[]) means.get(key);
            double[][] cov = (double[][]) covs.get(key);
            double weight = gmm.getWeight(key);
            System.out.println(key.toString() + ": w = " + (weight * 100) +
                               " ,rgn1 = " + mean[0] + ", rgn2 = " + mean[1] +
                               ", cov1 = " + cov[0][0] + ", cov2 = " + cov[1][1]);
          }
          boolean all_component_good = true;
          double allowed_dev = AbsRgnBiRelFeatures.NARROWED_DEV * 10;
          for (Iterator i = keys.iterator(); i.hasNext(); ) {
            Object key = i.next();
            double[] mean = (double[]) means.get(key);
            double[][] cov = (double[][]) covs.get(key);
            double weight = gmm.getWeight(key);
            if (Math.abs(mean[0] - Math.round(mean[0])) >
                allowed_dev ||
                Math.abs(mean[1] - Math.round(mean[1])) >
                allowed_dev||
                Math.sqrt(cov[0][0]) > allowed_dev ||
                Math.sqrt(cov[1][1]) > allowed_dev) {
              System.out.println(" bad >> " + key.toString() + ": w = " +
                                 (weight * 100) + " ,rgn1 = " + mean[0] +
                                 ", rgn2 = " + mean[1] +
                                 ", cov1 = " + cov[0][0] + ", cov2 = " + cov[1][1]);
              all_component_good = false;
              break;
            }
          }
          if (all_component_good) break;
          // increase the bi-relation component number to make them valid
          m++;
          // if increasing the component number doesn't help
          if (m > 2*M) break;
        }
        // fix the bi-rel parameter
        GaussianMixtureModel gmm = (GaussianMixtureModel) m_gmmModels.get(
            object, BIRELFEATURE_NAME);
        HashMap means = gmm.getComponentMeans();
        HashMap covs = gmm.getComponentCovariance();
        ArrayList keys = new ArrayList(means.keySet());
        Collections.sort(keys);
        System.out.println("fix " + BIRELFEATURE_NAME + " Gaussian mixture model:");
        double fixed_cov = AbsRgnBiRelFeatures.NARROWED_DEV *AbsRgnBiRelFeatures.NARROWED_DEV;
        for (Iterator i = keys.iterator(); i.hasNext(); ) {
          Object key = i.next();
          double[] mean = (double[]) means.get(key);
          double[][] cov = (double[][]) covs.get(key);
          mean[0] = (int)Math.round(mean[0]);
          mean[1] = (int)Math.round(mean[1]);
          cov[0][0] = fixed_cov;
          cov[1][1] = fixed_cov;
        }
        for (Iterator i = keys.iterator(); i.hasNext(); ) {
          Object key = i.next();
          double[] mean = (double[]) means.get(key);
          double[][] cov = (double[][]) covs.get(key);
          double weight = gmm.getWeight(key);
          System.out.println(key.toString() + ": w = " + (weight * 100) +
                             " ,rgn1 = " + mean[0] + ", rgn2 = " + mean[1] +
                             ", cov1 = " + cov[0][0] + ", cov2 = " + cov[1][1]);
        }

      }
      catch (ModelSingularException e) {
        e.printStackTrace();
      }
  }


  /**
   * Construct abstract region features
   * @param object
   * @param img_list
   */
  public void constructAbstractRegionFeatures(String object, LinkedList img_list) {

      System.out.println();
      System.out.println("bi-rel abstract region feature on \"" + object + "\"" + "\n");

      String feature = BIRELFEATURE_NAME;

      // first step : adding bi-relation feature representation
      for (Iterator i = img_list.iterator(); i.hasNext(); ) {
        String img = (String) i.next();
        LinkedList classified_rgns = getClusterLikelihood(m_imgDataCenter, m_featureList,
            img, object);
        AbsRgnBiRelFeatures bi_ft = new AbsRgnBiRelFeatures(classified_rgns);
        try {
          FeatureRepresentationList representation = bi_ft.represent(null, 0);
          m_imgDataCenter.setImgFeatureList(img, feature, representation);
        }
        catch (Exception e) {
          e.printStackTrace();
        }
      }
  }

  /**
   * Get the list of features used to construct feature vectors
   * @return
   */
  protected LinkedList getFeaturesForConstructVector() {
    LinkedList features = new LinkedList();
    features.addAll(m_featureList);
    assert(!features.contains(BIRELFEATURE_NAME));
    features.add(BIRELFEATURE_NAME);
    return features;
  }

  /**
   * Filter the training data.  This method is overrided to erease some
   * non-promising features
   * @param data
   * @param object
   * @param feature
   * @param posTrainImgs
   * @return
   */
  double[][] filterTrainingData(double[][] data, String object, String feature, LinkedList posTrainImgs) {

    if (feature != BIRELFEATURE_NAME) return data;
    double[][] pos_scores = getFeatureVectors(posTrainImgs, object, m_featureList);
    int comp_num = pos_scores[0].length;

//    double[][] bi_prob_sum = new double[comp_num][comp_num];
//    double[] prob_sum = new double[comp_num];
//    for (int img = 0; img<pos_scores.length; img++) {
//      double[] log_probs = pos_scores[img];
//      for (int i = 0; i < comp_num - 1; i++) {
//        for (int j = i + 1; j < comp_num; j++) {
//          bi_prob_sum[i][j] += Math.exp(log_probs[i] + log_probs[j]);
//        }
//      }
//      for (int i = 0; i < comp_num; i++) {
//        prob_sum[i] += Math.exp(log_probs[i]);
//      }
//    }
//    // average
//    for (int i = 0; i < comp_num - 1; i++) {
//      for (int j = i + 1; j < comp_num; j++) {
//        bi_prob_sum[i][j] = bi_prob_sum[i][j]/posTrainImgs.size();
//      }
//    }
//    for (int i = 0; i < comp_num; i++) {
//      prob_sum[i] = prob_sum[i]/posTrainImgs.size();
//    }
//    // print the ratio
//    // average
//    DoubleScoreHashMap pair_scores = new DoubleScoreHashMap();
//    for (int i = 0; i < comp_num - 1; i++) {
//      for (int j = i + 1; j < comp_num; j++) {
//        double ratio = bi_prob_sum[i][j]/prob_sum[i]/prob_sum[j];
//        pair_scores.putScore(new Dimension(i,j), ratio);
//      }
//    }
//    ArrayList sorted_pair = pair_scores.getObjectsByOrder();
//    HashSet kept_pair = new HashSet();
//    for (Iterator i=sorted_pair.iterator(); i.hasNext(); ) {
//      Dimension pair = (Dimension)i.next();
//      if (pair_scores.getScore(pair) > 1.2) {
//        kept_pair.add(pair);
//        System.out.println("(" + pair.width + "," + pair.height + ")=" + pair_scores.getScore(pair));
//        System.out.println("kept");
//      }
//    }
//    System.out.println("component pair kept = " + kept_pair.size());



    int[][] bi_pair_count = new int[comp_num][comp_num];
    for (int i=0; i<data.length; i++) {
      double[] d = data[i];
      int comp1 = (int) Math.round(d[0]);
      int comp2 = (int) Math.round(d[1]);
      bi_pair_count[comp1][comp2]++;
    }
    // count
    DoubleScoreHashMap pair_scores = new DoubleScoreHashMap();
    for (int i = 0; i < comp_num - 1; i++) {
      for (int j = i + 1; j < comp_num; j++) {
        double score = bi_pair_count[i][j];
        pair_scores.putScore(new Dimension(i,j), score);
      }
    }
    ArrayList sorted_pair = pair_scores.getObjectsByOrder();
    HashSet kept_pair = new HashSet();
    for (int i=sorted_pair.size()-1; i>sorted_pair.size()-1-5; i--) {
      Dimension pair = (Dimension)sorted_pair.get(i);
      kept_pair.add(pair);
      System.out.println("(" + pair.width + "," + pair.height + ")=" + pair_scores.getScore(pair));
      System.out.println("kept");
    }
    System.out.println("component pair kept = " + kept_pair.size());

    LinkedList kept_data = new LinkedList();
    for (int i=0; i<data.length; i++) {
      double[] d = data[i];
      int comp1 = (int)Math.round(d[0]);
      int comp2 = (int)Math.round(d[1]);
      if (kept_pair.contains(new Dimension(comp1, comp2))) {
        kept_data.add(d);
      }
    }
    System.out.println("data num after filter = " + data.length + "(" + ((double)kept_data.size()/data.length * 100) + "%)");
    data = yi.util.data.Conversion.to2DDoubleArray(kept_data);

    return data;
  }


//  /**
//   * Suppose we got the Mixture Gaussian models for every feature already.  Now
//   * it is the time to add the Mixture Gaussian model for bi-relations.
//   *
//   * The first step is adding bi-relation feature representation
//   * (AbsRgnBiRelFeatures) for each image in the image data center.  The second
//   * step is to learn the Gaussian Mixture model for this new feature.  The
//   * third step is to train the NN model which will consider the bi-relation
//   * feature.
//   *
//   * The new features will be added into the data center
//   *
//   * @param data_center
//   * @param features
//   * @param img_list
//   * @param obj
//   */
//  public void learnBiRelationModel(ImgDataCenter data_center, LinkedList features,
//                                   LinkedList img_list, String object) {
//    String feature = BIRELFEATURE_NAME;
//
//    // learn an additional gmm model for bi-relation
//    // seperate training set and verifying set
//    LinkedList posTrainImgs = new LinkedList();
//    LinkedList negTrainImgs = new LinkedList();
//    seperatePosImgsFromNegImgs(img_list, object, posTrainImgs, negTrainImgs);
//
//    // learn a gmm for "BiRel"
//    try {
//      learnGaussianMixtureModelOnFeature(object, feature, posTrainImgs);
//    }
//    catch (ModelSingularException e) {
//      e.printStackTrace();
//    }
//
//  }

//  /**
//   * Generate the bi-relation feature for the images
//   * @param data_center
//   * @param features
//   * @param img_list
//   * @param object
//   */
//  public void generateBiRelFeature(ImgDataCenter data_center,
//                                   LinkedList features,
//                                   LinkedList img_list, String object) {
//    String feature = BIRELFEATURE_NAME;
//
//    // first step : adding bi-relation feature representation
//    for (Iterator i = img_list.iterator(); i.hasNext(); ) {
//      String img = (String) i.next();
//      LinkedList classified_rgns = getClusterLikelihood(data_center, features,
//          img, object);
//      AbsRgnBiRelFeatures bi_ft = new AbsRgnBiRelFeatures(classified_rgns);
//      try {
//        FeatureRepresentationList representation = bi_ft.represent(null, 0);
//        data_center.setImgFeatureList(img, feature, representation);
//      }
//      catch (Exception e) {
//        e.printStackTrace();
//      }
//    }
//  }

  /**
   * For each region feature of the image, return all the classified abstract
   * regions
   * @param data_center
   * @param features
   * @param img
   * @param obj
   * @return linked list of ClassifiedAbsRgn
   */
  LinkedList getClusterLikelihood(ImgDataCenter data_center,
                                  LinkedList features, String img,
                                  String obj) {

//    System.out.println("getClusterLikelihood for " + img);

    int total_components = getTotalComponentNum(obj, features);

    LinkedList abs_rgns = new LinkedList();

    int component_count_base = 0; // the first feature begins from 0
                                  // the next will begin from the total count of the previous
    int f_index = 0;
    for (Iterator f = features.iterator(); f.hasNext(); f_index++) {
      String feature = (String) f.next();
//      System.out.println("consider " + feature);

      // get the feature representations (supposed to be region based representation)
      FeatureRepresentationList feature_data = data_center.getImgFeatureList(img, feature);

      // finds out how many Gaussian components
      GaussianMixtureModel gmm = (GaussianMixtureModel) m_gmmModels.get(
          obj, feature);
      int n_components = gmm.getRstNumComponents();
//      System.out.println("component# " + n_components);

      double minimum = ((Double)m_gmmUniformDensity.get(obj, feature)).doubleValue();

      for (Iterator d = feature_data.iterator(); d.hasNext(); ) {
        Object representation = d.next();
        if (!(representation instanceof EllipseFeatureRepresentation)) continue;
        EllipseFeatureRepresentation ellipse_rgn = (EllipseFeatureRepresentation)representation;
        // classify the data
        ClusterLikelihood cluster = getClusterLikelihood(ellipse_rgn.data, gmm,
            minimum);
        if (cluster != null) {
          abs_rgns.add(new ClassifiedAbsRgn(ellipse_rgn.region,
                                            cluster.clusterIndex + component_count_base,
                                            cluster.normDist));
        }
      }
      component_count_base += gmm.getRstNumComponents();
    } // for each feature

    return abs_rgns;
  }


  /**
   * See the background information in ClusterLikelihood.
   *
   * Return the cluster likelihood number of the given data.  The returned
   * number tells which cluster the data is classified as well as the likelihood
   *
   * @param data the data to classify
   * @param gmm the Gaussian mixture
   * @param min_log_prob the min log probability allowed on each Gaussian
   *        component.  If a data classified to its closest cluster i, but the
   *        density function returns a number even smaller than the defined
   *        minimum (which means the data is not close to any of the Gaussian
   *        component), then null will be returned.
   * @return
   */
  ClusterLikelihood getClusterLikelihood(double[] data, GaussianMixtureModel gmm,
      double minimum) {
    // classify the data to one of the Gaussian component
    Object cluster = gmm.classify(data);

    try {
      // check the density
      double density = gmm.getGaussianComponent(cluster).getDensity(data);
      if (density == 0 || density < minimum)
        return null;

      // calculate the distance between the data and the Gaussian component mean
      // (both are standardlized)
      double[] standardized = gmm.getGaussianComponent(cluster).standardize(data);
      double dist = yi.math.VectorUtil.lenth(standardized);
//      System.out.println("getClusterLikelihood: classified = " + cluster + "  " + density + "->" + dist);
      return new ClusterLikelihood(((Integer)cluster).intValue(), dist);
    } catch (GaussianModelSingularException e) {
      return null;
    }

  }

  /**
   * A region feature will be classified to one of the cluster (index of i).
   * Now that region will be represented by a cluster likelihood (type of double), l.
   * The "l" tells that the region was classified to cluster i (since it is close
   * to i than any other cluster index number) and the distance between l and i
   * will tell how likely the region feature is classified to cluster i.
   *
   * For example, the region feature d=(50, 100, 100) is classified to a Gaussian
   * cluster with mean of (75, 75, 75) and standard-dev. of (25, 25, 25).  First
   * the Gaussian distribution will be normalized, so the featuer will be
   * transfered as: d'=(d-mean)/sqrt(Sigma).  Now the norm of d' (|d'|) can be
   * used in a 1D standard normal distribution and tells how likely d belongs
   * to the cluster.  The number cluster likelihood l = |d'| * NARROW_DEV +/- i
   * ensures that l is close enough to i and it also represents how likely d
   * belongs to cluster i.
   *
   * In data structure of ClusterLikelihood, clusterIndex is i and normDist is |d'|
   */
  class ClusterLikelihood {
    int clusterIndex;
    double normDist;
    public ClusterLikelihood(int cluster_index, double norm_dist) {
      clusterIndex = cluster_index;
      normDist = norm_dist;
    }
  }

//  public static class Test {
//
//    public final static String USAGE = "Usage: [-d img_data_center_file] -m model_file -t training_image -i testing_img [-f feature] -o object [-r rst_file_name] ";
//
//    public static void main(String[] args) {
//
//      // init configuration
//      Configuration.init(args);
//
//      try {
//        // read in an StatModelAdapter object
//        String model_file = ReadArgu.getOption('m', args);
//        if (model_file == null) {
//          System.err.println(USAGE);
//          System.exit(1);
//        }
//
//        String path = Configuration.getParameter("LEARNING_PATH", "");
//
//        GMMNNWithBiRelAdapter sma = (GMMNNWithBiRelAdapter)yi.util.io.input.DIU_Ser.input(path + model_file);
//
//        // read in the datacenter
//        String data_center_name = ReadArgu.getOption('d', args, null);
//        ImgDataCenter data_center = null;
//        if (data_center_name != null) {
//          data_center = ImgDataCenter.createFromFile(path + data_center_name);
//        } else {
//          data_center = sma.getImgDataCente();
//        }
//
//        // object
//        String object = ReadArgu.getOption('o', args);
//        if (object == null) {
//          System.err.println(USAGE);
//          System.exit(1);
//        }
//
//        // get the training image list file name
//        String train_img_list_file = ReadArgu.getOption('t', args);
//
//        // image list
//        LinkedList train_img_list = null;
//        if (train_img_list_file != null) {
//          String[] images = yi.util.io.ReadArgu.readStrings(path + train_img_list_file);
//          train_img_list = yi.util.data.Conversion.asLinkedList(images);
//        } else {
//          System.err.println(USAGE);
//          System.exit(1);
//        }
//
//        // get the image list file name
//        String img_list_file = ReadArgu.getOption('i', args);
//
//        // image list
//        LinkedList img_list = null;
//        if (img_list_file != null) {
//          String[] images = yi.util.io.ReadArgu.readStrings(path + img_list_file);
//          img_list = yi.util.data.Conversion.asLinkedList(images);
//        } else {
//          System.err.println(USAGE);
//          System.exit(1);
//        }
//
//        // generate the new feature
//        LinkedList all_images = new LinkedList();
//        all_images.addAll(train_img_list);
//        all_images.addAll(img_list);
//        sma.generateBiRelFeature(data_center, sma.m_featureList, all_images, object);
//
//        // train on bi-rel
//        sma.learnBiRelationModel(data_center, sma.m_featureList, train_img_list, object);
//
//
//        sma.VERBOSE_IN_TEST = true;
//        StatTestRst rst = sma.test(img_list);
//
//        HashMap rpcs = rst.rpcAnalysis(data_center); // right positive curves
//
//
//        DoubleScoreHashMap roc_scores = rst.getROCScores(rpcs);
//        System.out.println("roc scores:");
//        System.out.println(roc_scores.toStringBykeyOrder());
//
//        Plot2DPanel pp = rst.drawROCs(rpcs);
//
//        String rst_output_file = ReadArgu.getOption('r', args);
//        if (rst_output_file != null) {
//          pp.setFixedBounds(0,0,1);
//          pp.setFixedBounds(1,0,1);
//          pp.invalidate();
//          pp.toGraphicFile(new java.io.File(path + rst_output_file + ".png"));
//        }
//
//      } catch (Exception e) {
//        e.printStackTrace();
//      }
//    }
//  }

}