Added recursive grid opt

MatlabGP/+optim/RecursiveGrid.m

@@ -0,0 +1,41 @@
+function [x0,fmin,xv,fv] = RecursiveGrid(f,M,N,lb,ub,vectorize)
+
+if nargin<6
+    vectorize = 0;
+end
+
+dx = (ub-lb)/2;
+
+x0 = 0.5*(lb+ub);
+
+for i = 1:M
+
+    lb1 = max(x0 - dx,lb);
+    ub1 = min(x0 + dx,ub);
+
+    x = lb1 + (ub1 - lb1).*lhsdesign(N,length(lb));
+
+    if vectorize
+        fn = f(x);
+    else
+        for j = 1:N
+            fn(j) = f(x(j,:));
+        end
+    end
+
+    [~,imin] = min(fn);
+
+    x0 = x(imin,:);
+    xv(i,:) = x0;
+
+    dx = dx/5;
+
+    fv(i) = fn(imin);
+
+end
+
+fmin = fn(imin);
+
+
+
+end

MatlabGP/GP.m

@@ -260,34 +260,35 @@
 
             func = @(x) obj.LL(x,regress,ntm);
 
-            xxt = tlb + (tub - tlb).*lhsdesign(200*length(tlb),length(tlb));
-
-            for ii = 1:size(xxt,1)
-                LL(ii) = func(xxt(ii,:));
-            end
-
-            LL = exp(1 + LL - max(LL));
-
-            theta = sum(xxt.*LL')/sum(LL);
-
-            % for i = 1:2
-            %     %tx0 = tlb + (tub - tlb).*rand(1,length(tlb));
-            % 
-            %     %opts = optimoptions('fmincon','SpecifyObjectiveGradient',true,'Display','none');
-            % 
-            %     %[theta{i},val(i)] = fmincon(func,tx0,[],[],[],[],tlb,tub,[],opts);
-            % 
-            %     opts.TolMesh = 1e-2;
-            %     opts.TolFun = 1e-2;
+            % xxt = tlb + (tub - tlb).*lhsdesign(200*length(tlb),length(tlb));
             % 
-            %     [theta{i},val(i)] = bads(func,tx0,tlb,tub,[],[],[],opts);
-            %     %[theta{i},val(i)] = VSGD(func,tx0,'lr',0.02,'lb',tlb,'ub',tub,'gamma',0.0001,'iters',20,'tol',1*10^(-4));
-            %     %[theta{i},val(i)] = optim.minimizebnd(func,tx0,tlb,tub,1,0);
+            % for ii = 1:size(xxt,1)
+            %     LL(ii) = func(xxt(ii,:));
             % end
             % 
-            % [mval,i] = min(val);
+            % LL = exp(1 + LL - max(LL));
             % 
-            % theta = theta{i};
+            % theta = sum(xxt.*LL')/sum(LL);
+
+            for i = 1:2
+                %tx0 = tlb + (tub - tlb).*rand(1,length(tlb));
+
+                %opts = optimoptions('fmincon','SpecifyObjectiveGradient',true,'Display','none');
+
+                %[theta{i},val(i)] = fmincon(func,tx0,[],[],[],[],tlb,tub,[],opts);
+
+                %opts.TolMesh = 1e-2;
+                %opts.TolFun = 1e-2;
+                [theta{i},val(i)] = optim.RecursiveGrid(func,6,20,tlb,tub);
+
+                %[theta{i},val(i)] = bads(func,tx0,tlb,tub,[],[],[],opts);
+                %[theta{i},val(i)] = VSGD(func,tx0,'lr',0.02,'lb',tlb,'ub',tub,'gamma',0.0001,'iters',20,'tol',1*10^(-4));
+                %[theta{i},val(i)] = optim.minimizebnd(func,tx0,tlb,tub,1,0);
+            end
+
+            [mval,i] = min(val);
+
+            theta = theta{i};
 
             if regress
                 obj.kernel.signn = theta(end);

MatlabGP/docs/testNN.asv

@@ -0,0 +1,71 @@
+
+clear all
+close all
+clc
+
+xx = [0;lhsdesign(30,1);1];
+yy = forr(xx,0);
+
+xmesh = linspace(0,1,100)';
+ymesh = forr(xmesh,0);
+
+layers{1} = NN.FF(1,6);
+layers{2} = NN.FF(6,10);
+layers{3} = NN.FF(10,3);
+
+acts{1} = NN.SWISH(1);
+acts{2} = NN.SNAKE(0.1);
+
+lss = NN.MSE();
+
+
+nnet = NN.NN(layers,acts,lss);
+
+%%
+for ii = 1:20
+    tic
+    [nnet2{ii},fval(ii),xv,fv] = nnet.train(xx,yy);
+    toc
+end
+
+%%
+
+for j = 1:length(xmesh)
+    %yp1(:,j) = nnet.predict(xmesh(j,:)');
+    yp2(:,j) = nnet2.predict(xmesh(j));
+end
+
+%%
+figure
+plot(fv,'.')
+set(gca,'yscale','log')
+set(gca,'xscale','log')
+
+figure
+plot(xmesh,ymesh)
+hold on
+%plot(xmesh,yp1)
+plot(xmesh,yp2)
+plot(xx,yy,'x')
+
+%%
+
+function y = forr(x,dx)
+
+nx = length(x);
+
+A = 0.5; B = 10; C = -5;
+
+for i = 1:nx
+    if x(i)<0.45
+        y(i,1) = (6*x(i)-2).^2.*sin(12*x(i)-4);
+    else
+        y(i,1) = (6*x(i)-2).^2.*sin(12*x(i)-4)+dx;
+    end
+
+    y(i,2) = 0.4*(6*x(i)-2).^2.*sin(12*x(i)-4)-x(i)-1;
+    y(i,3) = A*(6*x(i)-2).^2.*sin(12*x(i)-4)+B*(x(i)-0.5)-C;
+end
+
+end
+

MatlabGP/docs/testNN.m

@@ -9,28 +9,31 @@
 xmesh = linspace(0,1,100)';
 ymesh = forr(xmesh,0);
 
-layers{1} = NN.FF(1,5);
-layers{2} = NN.FF(5,10);
+layers{1} = NN.FF(1,6);
+layers{2} = NN.FF(6,10);
 layers{3} = NN.FF(10,3);
 
 acts{1} = NN.SNAKE(1);
-acts{2} = NN.SWISH(1);
+acts{2} = NN.SNAKE(1);
 
 lss = NN.MSE();
 
 
 nnet = NN.NN(layers,acts,lss);
 
 %%
-tic
-[nnet2,fval,xv,fv] = nnet.train(xx,yy);
-toc
+for ii = 1:20
+    tic
+    [nnet2{ii},fval(ii),xv,fv] = nnet.train(xx,yy);
+    toc
+end
 
 %%
-
-for j = 1:length(xmesh)
-    %yp1(:,j) = nnet.predict(xmesh(j,:)');
-    yp2(:,j) = nnet2.predict(xmesh(j));
+for ii = 1:20
+    for j = 1:length(xmesh)
+        %yp1(:,j) = nnet.predict(xmesh(j,:)');
+        yp2(:,j,ii) = nnet2{ii}.predict(xmesh(j));
+    end
 end
 
 %%
@@ -43,7 +46,9 @@
 plot(xmesh,ymesh)
 hold on
 %plot(xmesh,yp1)
-plot(xmesh,yp2)
+plot(xmesh,squeeze(mean(yp2,3)))
+plot(xmesh,squeeze(mean(yp2,3))+2*squeeze(std(yp2,[],3)),'--')
+plot(xmesh,squeeze(mean(yp2,3))-2*squeeze(std(yp2,[],3)),'--')
 plot(xx,yy,'x')
 
 %%