load 'MNIST_subset.mat';

%% Normalization
for i=1:size(tr_feats,1)
    tr_feats(i,:)=tr_feats(i,:)/sqrt(sum(tr_feats(i,:).^2));
end
for i=1:size(te_feats,1)
    te_feats(i,:)=te_feats(i,:)/sqrt(sum(te_feats(i,:).^2));
end

%% PCA: Try to include the following PCA code or not. Compare the results.



% mean_tr_feats=mean(tr_feats);
% normalized_tr_feats=tr_feats-repmat(mean_tr_feats,1000,1);
% cov=normalized_tr_feats'*normalized_tr_feats;
% [V,D]=eig(cov);
% [value,index]=sort(diag(D),'descend');
% V_proj=V(:,index(1:100));       % only retain the top 100 dimensions
% for i=1:10
%     subplot(2,5,i);
%     tmp=reshape(V_proj(:,i),28,28);
%     tmp=tmp';
%     imshow((tmp-min(tmp(:)))/(max(tmp(:)-min(tmp(:)))));
% end
% tr_feats=normalized_tr_feats*V_proj;
% te_feats=(te_feats-repmat(mean_tr_feats,10000,1))*V_proj;




%% gaussian distribution assumption
%% estimate the mean and standard deviation for each dimension
for i=0:9
    index=find(tr_label==i);
    digits_feats=tr_feats(index,:);
    digits_mean(i+1,:)=mean(digits_feats);
    digits_std(i+1,:)=std(digits_feats);
end

digits_std(digits_std==0)=10000;  % avoid the influence of these definite bins

%% testing
for i=1:1000    % only test the first 1000 samples
    sample=te_feats(i,:);
    sample=repmat(sample,10,1);
    prob=1./digits_std.*exp(-(sample-digits_mean).^2./(2*digits_std.^2)); % gaussian pdf
    prob=prob./repmat(sum(prob),10,1);
    prob=sum(prob,2);
    [value,predicted_label(i)]=max(prob);
    if mod(i,100)==0
        fprintf('.');
    end
end

fprintf('\n The classification accuracy for Naive Bayesian Classifier is %f.\n',sum(predicted_label'==te_label(1:1000)+1)/1000);



%% K-NN
clear all;
load 'MNIST_subset.mat';

%% Normalization
for i=1:size(tr_feats,1)
    tr_feats(i,:)=tr_feats(i,:)/sqrt(sum(tr_feats(i,:).^2));
end
for i=1:size(te_feats,1)
    te_feats(i,:)=te_feats(i,:)/sqrt(sum(te_feats(i,:).^2));
end

K=5;
for i=1:1000    % only test the first 1000 samples
    dist=sum((tr_feats-repmat(te_feats(i,:),1000,1)).^2,2);
    [value,index]=sort(dist,'ascend');
    predicted_label(i)=mode(tr_label(index(1:K)));
    if mod(i,50)==0
        fprintf('.');
    end
end
fprintf('\n The classification accuracy for 5-NN is %f.\n',sum(predicted_label'==te_label(1:1000))/1000);



