001/*-
002 *******************************************************************************
003 * Copyright (c) 2011, 2016 Diamond Light Source Ltd.
004 * All rights reserved. This program and the accompanying materials
005 * are made available under the terms of the Eclipse Public License v1.0
006 * which accompanies this distribution, and is available at
007 * http://www.eclipse.org/legal/epl-v10.html
008 *
009 * Contributors:
010 *    Peter Chang - initial API and implementation and/or initial documentation
011 *******************************************************************************/
012
013package org.eclipse.january.dataset;
014
015import java.io.Serializable;
016import java.lang.annotation.Annotation;
017import java.lang.reflect.Array;
018import java.lang.reflect.Field;
019import java.util.ArrayList;
020import java.util.Arrays;
021import java.util.Collection;
022import java.util.concurrent.ConcurrentMap;
023import java.util.concurrent.ConcurrentHashMap;
024import java.util.List;
025import java.util.Map;
026
027import org.eclipse.january.DatasetException;
028import org.eclipse.january.MetadataException;
029import org.eclipse.january.metadata.Dirtiable;
030import org.eclipse.january.metadata.ErrorMetadata;
031import org.eclipse.january.metadata.IMetadata;
032import org.eclipse.january.metadata.MetadataFactory;
033import org.eclipse.january.metadata.MetadataType;
034import org.eclipse.january.metadata.Reshapeable;
035import org.eclipse.january.metadata.Sliceable;
036import org.eclipse.january.metadata.Transposable;
037import org.slf4j.Logger;
038import org.slf4j.LoggerFactory;
039
040/**
041 * Common base for both lazy and normal dataset implementations
042 */
043public abstract class LazyDatasetBase implements ILazyDataset, Serializable {
044
045        private static final long serialVersionUID = 767926846438976050L;
046
047        protected static final Logger logger = LoggerFactory.getLogger(LazyDatasetBase.class);
048
049        protected static boolean catchExceptions;
050
051        static {
052                /**
053                 * Boolean to set to true if running jython scripts that utilise ScisoftPy in IDE
054                 */
055                try {
056                        catchExceptions = Boolean.getBoolean("run.in.eclipse");
057                } catch (SecurityException e) {
058                        // set a default for when the security manager does not allow access to the requested key
059                        catchExceptions = false;
060                }
061        }
062
063        transient private boolean dirty = true; // indicate dirty state of metadata
064        protected String name = "";
065
066        /**
067         * The shape or dimensions of the dataset
068         */
069        protected int[] shape;
070
071        protected ConcurrentMap<Class<? extends MetadataType>, List<MetadataType>> metadata = null;
072
073        /**
074         * @return type of dataset item
075         */
076        abstract public int getDType();
077
078        @Override
079        public Class<?> getElementClass() {
080                return DTypeUtils.getElementClass(getDType());
081        }
082
083        @Override
084        public LazyDatasetBase clone() {
085                return null;
086        }
087
088        @Override
089        public boolean equals(Object obj) {
090                if (this == obj) {
091                        return true;
092                }
093                if (obj == null) {
094                        return false;
095                }
096                if (!getClass().equals(obj.getClass())) {
097                        return false;
098                }
099
100                LazyDatasetBase other = (LazyDatasetBase) obj;
101                if (getDType() != other.getDType()) {
102                        return false;
103                }
104                if (getElementsPerItem() != other.getElementsPerItem()) {
105                        return false;
106                }
107                if (!Arrays.equals(shape, other.shape)) {
108                        return false;
109                }
110                return true;
111        }
112
113        @Override
114        public int hashCode() {
115                int hash = getDType() * 17 + getElementsPerItem();
116                int rank = shape.length;
117                for (int i = 0; i < rank; i++) {
118                        hash = hash*17 + shape[i];
119                }
120                return hash;
121        }
122
123        @Override
124        public String getName() {
125                return name;
126        }
127
128        @Override
129        public void setName(String name) {
130                this.name = name;
131        }
132
133        @Override
134        public int[] getShape() {
135                return shape.clone();
136        }
137
138        @Override
139        public int getRank() {
140                return shape.length;
141        }
142
143        /**
144         * This method allows anything that dirties the dataset to clear various metadata values
145         * so that the other methods can work correctly.
146         * @since 2.1
147         */
148        public void setDirty() {
149                dirty = true;
150        }
151
152        /**
153         * Find first sub-interface of (or class that directly implements) MetadataType
154         * @param clazz
155         * @return sub-interface
156         * @exception IllegalArgumentException when given class is {@link MetadataType} or an anonymous sub-class of it
157         */
158        @SuppressWarnings("unchecked")
159        public static Class<? extends MetadataType> findMetadataTypeSubInterfaces(Class<? extends MetadataType> clazz) {
160                if (clazz.equals(MetadataType.class)) {
161                        throw new IllegalArgumentException("Cannot accept MetadataType");
162                }
163
164                if (clazz.isInterface()) {
165                        return clazz;
166                }
167
168                if (clazz.isAnonymousClass()) { // special case
169                        Class<?> s = clazz.getSuperclass();
170                        if (!s.equals(Object.class)) {
171                                // only use super class if it is not an anonymous class of an interface
172                                clazz = (Class<? extends MetadataType>) s;
173                        }
174                }
175
176                for (Class<?> c : clazz.getInterfaces()) {
177                        if (c.equals(MetadataType.class)) {
178                                if (clazz.isAnonymousClass()) {
179                                        throw new IllegalArgumentException("Cannot accept anonymous subclasses of MetadataType");
180                                }
181                                return clazz;
182                        }
183                        if (MetadataType.class.isAssignableFrom(c)) {
184                                return (Class<? extends MetadataType>) c;
185                        }
186                }
187
188                Class<?> c = clazz.getSuperclass(); // Naughty: someone has sub-classed a metadata class
189                if (c != null) {
190                        return findMetadataTypeSubInterfaces((Class<? extends MetadataType>) c);
191                }
192
193                logger.error("Somehow the search for metadata type interface ended in a bad place");
194                assert false; // should not be able to get here!!!
195                return null;
196        }
197
198        @Override
199        public void setMetadata(MetadataType metadata) {
200                addMetadata(metadata, true);
201        }
202
203        @Override
204        public void addMetadata(MetadataType metadata) {
205                addMetadata(metadata, false);
206        }
207
208        private synchronized void addMetadata(MetadataType metadata, boolean clear) {
209                if (metadata == null) {
210                        return;
211                }
212
213                if (this.metadata == null) {
214                        this.metadata = new ConcurrentHashMap<Class<? extends MetadataType>, List<MetadataType>>();
215                }
216
217                Class<? extends MetadataType> clazz = findMetadataTypeSubInterfaces(metadata.getClass());
218                if (!this.metadata.containsKey(clazz)) {
219                        this.metadata.put(clazz, new ArrayList<MetadataType>());
220                } else if (clear) {
221                        this.metadata.get(clazz).clear();
222                }
223                this.metadata.get(clazz).add(metadata);
224
225                // add for special case of sub-interfaces of IMetadata
226                if (!IMetadata.class.equals(clazz) && IMetadata.class.isAssignableFrom(clazz)) {
227                        clazz = IMetadata.class;
228                        if (!this.metadata.containsKey(clazz)) {
229                                this.metadata.put(clazz, new ArrayList<MetadataType>());
230                        } else if (clear) {
231                                this.metadata.get(clazz).clear();
232                        }
233                        this.metadata.get(clazz).add(metadata);
234                }
235        }
236
237        @Override
238        @Deprecated
239        public synchronized IMetadata getMetadata() {
240                return getFirstMetadata(IMetadata.class);
241        }
242
243        @SuppressWarnings("unchecked")
244        @Override
245        public synchronized <S extends MetadataType, T extends S> List<S> getMetadata(Class<T> clazz) throws MetadataException {
246                if (metadata == null) {
247                        dirty = false;
248                        return null;
249                }
250
251                if (dirty) {
252                        dirtyMetadata();
253                        dirty = false;
254                }
255
256                if (clazz == null) {
257                        List<S> all = new ArrayList<S>();
258                        for (Class<? extends MetadataType> c : metadata.keySet()) {
259                                all.addAll((Collection<S>) metadata.get(c));
260                        }
261                        return all;
262                }
263
264                return (List<S>) metadata.get(findMetadataTypeSubInterfaces(clazz));
265        }
266
267        @Override
268        public synchronized <S extends MetadataType, T extends S> S getFirstMetadata(Class<T> clazz) {
269                try {
270                        List<S> ml = getMetadata(clazz);
271                        if (ml == null) {
272                                return null;
273                        }
274                        for (S t : ml) {
275                                if (clazz.isInstance(t)) {
276                                        return t;
277                                }
278                        }
279                } catch (Exception e) {
280                        logger.error("Get metadata failed!",e);
281                }
282
283                return null;
284        }
285
286        @Override
287        public synchronized void clearMetadata(Class<? extends MetadataType> clazz) {
288                if (metadata == null) {
289                        return;
290                }
291
292                if (clazz == null) {
293                        metadata.clear();
294                        return;
295                }
296
297                List<MetadataType> list = metadata.get(findMetadataTypeSubInterfaces(clazz));
298                if( list != null) {
299                        list.clear();
300                }
301        }
302
303        /**
304         * @since 2.0
305         */
306        protected synchronized ConcurrentMap<Class<? extends MetadataType>, List<MetadataType>> copyMetadata() {
307                return copyMetadata(metadata);
308        }
309
310        /**
311         * @since 2.0
312         */
313        protected static ConcurrentMap<Class<? extends MetadataType>, List<MetadataType>> copyMetadata(Map<Class<? extends MetadataType>, List<MetadataType>> metadata) {
314                if (metadata == null) {
315                        return null;
316                }
317
318                ConcurrentHashMap<Class<? extends MetadataType>, List<MetadataType>> map = new ConcurrentHashMap<Class<? extends MetadataType>, List<MetadataType>>();
319                copyMetadata(metadata, map);
320                return map;
321        }
322
323        private static void copyMetadata(Map<Class<? extends MetadataType>, List<MetadataType>> inMetadata,
324                        Map<Class<? extends MetadataType>, List<MetadataType>> outMetadata) {
325                for (Class<? extends MetadataType> c : inMetadata.keySet()) {
326                        List<MetadataType> l = inMetadata.get(c);
327                        List<MetadataType> nl = new ArrayList<MetadataType>(l.size());
328                        outMetadata.put(c, nl);
329                        for (MetadataType m : l) {
330                                if (m == null || isMetadataDirty(m)) { // skip dirty metadata
331                                        continue;
332                                }
333                                nl.add(m.clone());
334                        }
335                }
336        }
337
338        protected void restoreMetadata(Map<Class<? extends MetadataType>, List<MetadataType>> oldMetadata) {
339                copyMetadata(oldMetadata, metadata);
340        }
341
342        /**
343         * @since 2.2
344         */
345        protected static ConcurrentMap<Class<? extends MetadataType>, List<MetadataType>> getMetadataMap(ILazyDataset a, boolean clone) {
346                List<MetadataType> all = null;
347                try {
348                        all = a.getMetadata(null);
349                } catch (Exception e) {
350                }
351                if (all == null) {
352                        return null;
353                }
354
355                ConcurrentMap<Class<? extends MetadataType>, List<MetadataType>> map = new ConcurrentHashMap<Class<? extends MetadataType>, List<MetadataType>>();
356
357                for (MetadataType m : all) {
358                        if (m == null || isMetadataDirty(m)) { // skip dirty metadata
359                                continue;
360                        }
361                        Class<? extends MetadataType> c = findMetadataTypeSubInterfaces(m.getClass());
362                        List<MetadataType> l = map.get(c);
363                        if (l == null) {
364                                l = new ArrayList<MetadataType>();
365                                map.put(c, l);
366                        }
367                        if (clone) {
368                                m = m.clone();
369                        }
370                        l.add(m);
371                }
372                return map;
373        }
374
375        private static boolean isMetadataDirty(MetadataType m) {
376                Class<? extends MetadataType> c = m.getClass();
377                for (Field f : c.getDeclaredFields()) {
378                        if (f.isAnnotationPresent(Dirtiable.class)) {
379                                Class<?> t = f.getType();
380                                if (t.equals(boolean.class) || t.equals(Boolean.class)) {
381                                        try {
382                                                f.setAccessible(true);
383                                                Object o = f.get(m);
384                                                if (o.equals(true)) {
385                                                        return true;
386                                                }
387                                        } catch (Exception e) {
388                                                logger.debug("Could not retrieve value of dirty variable: {}", c.getCanonicalName(), e);
389                                        }
390                                }
391                        }
392                }
393
394                return false;
395        }
396
397        interface MetadatasetAnnotationOperation {
398                /**
399                 * Process value of given field
400                 * <p>
401                 * When the field is not a container then the returned value
402                 * may replace the old value
403                 * @param f given field
404                 * @param o value of field
405                 * @return transformed field
406                 */
407                Object processField(Field f, Object o);
408
409                /**
410                 * @return annotated class
411                 */
412                Class<? extends Annotation> getAnnClass();
413
414                /**
415                 * @param axis
416                 * @return number of dimensions to insert or remove
417                 */
418                int change(int axis);
419
420                /**
421                 * 
422                 * @return rank or -1 to match
423                 */
424                int getNewRank();
425
426                /**
427                 * Run on given lazy dataset
428                 * @param lz
429                 * @return 
430                 */
431                ILazyDataset run(ILazyDataset lz);
432        }
433
434        class MdsSlice implements MetadatasetAnnotationOperation {
435                private boolean asView;
436                private SliceND slice;
437                private int[] oShape;
438                private long oSize;
439
440                public MdsSlice(boolean asView, SliceND slice) {
441                        this.asView = asView;
442                        this.slice = slice;
443                        oShape = slice.getSourceShape();
444                        oSize = ShapeUtils.calcLongSize(oShape);
445                }
446
447                @Override
448                public Object processField(Field field, Object o) {
449                        return o;
450                }
451
452                @Override
453                public Class<? extends Annotation> getAnnClass() {
454                        return Sliceable.class;
455                }
456
457                @Override
458                public int change(int axis) {
459                        return 0;
460                }
461
462                @Override
463                public int getNewRank() {
464                        return -1;
465                }
466
467                @Override
468                public ILazyDataset run(ILazyDataset lz) {
469                        int rank = lz.getRank();
470                        if (slice.getStart().length != rank) {
471                                throw new IllegalArgumentException("Slice rank does not match dataset!");
472                        }
473
474                        int[] shape = lz.getShape();
475                        SliceND nslice;
476                        if (!ShapeUtils.areShapesBroadcastCompatible(oShape, shape)) {
477                                nslice = new SliceND(shape);
478                                for (int i = 0; i < rank; i++) {
479                                        int s = shape[i];
480                                        int os = oShape[i];
481                                        if (s >= os) {
482                                                nslice.setSlice(i, 0, os, 1);
483                                        } else if (s == 1) {
484                                                nslice.setSlice(i, 0, 1, 1);
485                                        } else {
486                                                throw new IllegalArgumentException("Sliceable dataset has non-unit dimension less than host!");
487                                        }
488                                }
489                                lz = lz.getSliceView(nslice);
490                        }
491                        if (lz.getSize() == oSize) {
492                                nslice = slice;
493                        } else {
494                                nslice = slice.clone();
495                                for (int i = 0; i < rank; i++) {
496                                        int s = shape[i];
497                                        if (s >= oShape[i]) {
498                                                continue;
499                                        } else if (s == 1) {
500                                                nslice.setSlice(i, 0, 1, 1);
501                                        } else {
502                                                throw new IllegalArgumentException("Sliceable dataset has non-unit dimension less than host!");
503                                        }
504                                }
505                        }
506
507                        if (asView || (lz instanceof IDataset)) {
508                                return lz.getSliceView(nslice);
509                        }
510                        try {
511                                return lz.getSlice(nslice);
512                        } catch (DatasetException e) {
513                                logger.error("Could not slice dataset in metadata", e);
514                                return null;
515                        }
516                }
517        }
518
519        class MdsReshape implements MetadatasetAnnotationOperation {
520                private boolean matchRank;
521                private int[] oldShape;
522                private int[] newShape;
523                boolean onesOnly;
524                int[] differences;
525
526                /*
527                 * if only ones then record differences (insertions and deletions)
528                 * 
529                 * if shape changing, find broadcasted dimensions and disallow
530                 * merging that include those dimensions
531                 */
532                public MdsReshape(final int[] oldShape, final int[] newShape) {
533                        this.oldShape = oldShape;
534                        this.newShape = newShape;
535                        differences = null;
536                }
537
538                @Override
539                public Object processField(Field field, Object o) {
540                        Annotation a = field.getAnnotation(Reshapeable.class);
541                        if (a != null) { // cannot be null
542                                matchRank = ((Reshapeable) a).matchRank();
543                        }
544                        return o;
545                }
546
547                @Override
548                public Class<? extends Annotation> getAnnClass() {
549                        return Reshapeable.class;
550                }
551
552                @Override
553                public int change(int axis) {
554                        if (matchRank) {
555                                if (differences == null) {
556                                        init();
557                                }
558
559                                if (onesOnly) {
560                                        return differences[axis];
561                                }
562                                throw new UnsupportedOperationException("TODO support other shape operations");
563                        }
564                        return 0;
565                }
566
567                @Override
568                public int getNewRank() {
569                        return matchRank ? newShape.length : -1;
570                }
571
572                private void init() {
573                        int or = oldShape.length - 1;
574                        int nr = newShape.length - 1;
575                        if (or < 0 || nr < 0) { // zero-rank shapes
576                                onesOnly = true;
577                                differences = new int[1];
578                                differences[0] = or < 0 ? nr + 1 : or + 1;
579                                return;
580                        }
581                        int ob = 0;
582                        int nb = 0;
583                        onesOnly = true;
584                        do {
585                                while (oldShape[ob] == 1 && ob < or) {
586                                        ob++; // next non-unit dimension
587                                }
588                                while (newShape[nb] == 1 && nb < nr) {
589                                        nb++;
590                                }
591                                if (oldShape[ob++] != newShape[nb++]) {
592                                        onesOnly = false;
593                                        break;
594                                }
595                        } while (ob <= or && nb <= nr);
596
597                        ob = 0;
598                        nb = 0;
599                        differences = new int[or + 2];
600                        if (onesOnly) {
601                                // work out unit dimensions removed from or add to old
602                                int j = 0;
603                                do {
604                                        if (oldShape[ob] != 1 && newShape[nb] != 1) {
605                                                ob++;
606                                                nb++;
607                                        } else {
608                                                while (oldShape[ob] == 1 && ob < or) {
609                                                        ob++;
610                                                        differences[j]--;
611                                                }
612                                                while (newShape[nb] == 1 && nb < nr) {
613                                                        nb++;
614                                                        differences[j]++;
615                                                }
616                                        }
617                                        j++;
618                                } while (ob <= or && nb <= nr && j <= or);
619                                while (ob <= or && oldShape[ob] == 1) {
620                                        ob++;
621                                        differences[j]--;
622                                }
623                                while (nb <= nr && newShape[nb] == 1) {
624                                        nb++;
625                                        differences[j]++;
626                                }
627                        } else {
628                                if (matchRank) {
629                                        logger.error("Combining dimensions is currently not supported");
630                                        throw new IllegalArgumentException("Combining dimensions is currently not supported");
631                                }
632                                // work out mapping: contiguous dimensions can be grouped or split
633                                while (ob <= or && nb <= nr) {
634                                        int ol = oldShape[ob];
635                                        while (ol == 1 && ol <= or) {
636                                                ob++;
637                                                ol = oldShape[ob];
638                                        }
639                                        int oe = ob + 1;
640                                        int nl = newShape[nb];
641                                        while (nl == 1 && nl <= nr) {
642                                                nb++;
643                                                nl = newShape[nb];
644                                        }
645                                        int ne = nb + 1;
646                                        if (ol < nl) {
647                                                differences[ob] = 1;
648                                                do { // case where new shape combines several dimensions into one dimension
649                                                        if (oe == (or + 1)) {
650                                                                break;
651                                                        }
652                                                        differences[oe] = 1;
653                                                        ol *= oldShape[oe++];
654                                                } while (ol < nl);
655                                                differences[oe - 1] = oe - ob; // signal end with difference
656                                                if (nl != ol) {
657                                                        logger.error("Single dimension is incompatible with subshape");
658                                                        throw new IllegalArgumentException("Single dimension is incompatible with subshape");
659                                                }
660                                        } else if (ol > nl) {
661                                                do { // case where new shape spreads single dimension over several dimensions
662                                                        if (ne == (nr + 1)) {
663                                                                break;
664                                                        }
665                                                        nl *= newShape[ne++];
666                                                } while (nl < ol);
667                                                if (nl != ol) {
668                                                        logger.error("Subshape is incompatible with single dimension");
669                                                        throw new IllegalArgumentException("Subshape is incompatible with single dimension");
670                                                }
671
672                                        }
673
674                                        ob = oe;
675                                        nb = ne;
676                                }
677
678                        }
679                }
680
681                @Override
682                public ILazyDataset run(ILazyDataset lz) {
683                        if (differences == null) {
684                                init();
685                        }
686
687                        int[] lshape = lz.getShape();
688                        if (Arrays.equals(newShape, lshape)) {
689                                return lz;
690                        }
691                        int or = lz.getRank();
692                        int nr = newShape.length;
693                        int[] nshape = new int[nr];
694                        Arrays.fill(nshape, 1);
695                        if (onesOnly) {
696                                // ignore omit removed dimensions
697                                for (int i = 0, si = 0, di = 0; i < (or+1) && si <= or && di < nr; i++) {
698                                        int c = differences[i];
699                                        if (c == 0) {
700                                                nshape[di++] = lshape[si++];
701                                        } else if (c > 0) {
702                                                while (c-- > 0 && di < nr) {
703                                                        di++;
704                                                }
705                                        } else if (c < 0) {
706                                                si -= c; // remove dimensions by skipping forward in source array
707                                        }
708                                }
709                        } else {
710                                boolean[] broadcast = new boolean[or];
711                                for (int ob = 0; ob < or; ob++) {
712                                        broadcast[ob] = oldShape[ob] != 1 && lshape[ob] == 1;
713                                }
714                                int osize = lz.getSize();
715
716                                // cannot do 3x5x... to 15x... if metadata is broadcasting (i.e. 1x5x...)
717                                int ob = 0;
718                                int nsize = 1;
719                                for (int i = 0; i < nr; i++) {
720                                        if (ob < or && broadcast[ob]) {
721                                                if (differences[ob] != 0) {
722                                                        logger.error("Metadata contains a broadcast axis which cannot be reshaped");
723                                                        throw new IllegalArgumentException("Metadata contains a broadcast axis which cannot be reshaped");
724                                                }
725                                        } else {
726                                                nshape[i] = nsize < osize ? newShape[i] : 1;
727                                        }
728                                        nsize *= nshape[i];
729                                        ob++;
730                                }
731                        }
732
733                        ILazyDataset nlz = lz.getSliceView();
734                        if (lz instanceof Dataset) {
735                                nlz = ((Dataset) lz).reshape(nshape);
736                        } else {
737                                nlz = lz.getSliceView();
738                                nlz.setShape(nshape);
739                        }
740                        return nlz;
741                }
742        }
743
744        class MdsTranspose implements MetadatasetAnnotationOperation {
745                int[] map;
746
747                public MdsTranspose(final int[] axesMap) {
748                        map = axesMap;
749                }
750
751                @SuppressWarnings({ "rawtypes", "unchecked" })
752                @Override
753                public Object processField(Field f, Object o) {
754                        // reorder arrays and lists according the axes map
755                        if (o.getClass().isArray()) {
756                                int l = Array.getLength(o);
757                                if (l == map.length) {
758                                        Object narray = Array.newInstance(o.getClass().getComponentType(), l);
759                                        for (int i = 0; i < l; i++) {
760                                                Array.set(narray, i, Array.get(o, map[i]));
761                                        }
762                                        for (int i = 0; i < l; i++) {
763                                                Array.set(o, i, Array.get(narray, i));
764                                        }
765                                }
766                        } else if (o instanceof List<?>) {
767                                List list = (List) o;
768                                int l = list.size();
769                                if (l == map.length) {
770                                        Object narray = Array.newInstance(o.getClass().getComponentType(), l);
771                                        for (int i = 0; i < l; i++) {
772                                                Array.set(narray, i, list.get(map[i]));
773                                        }
774                                        list.clear();
775                                        for (int i = 0; i < l; i++) {
776                                                list.add(Array.get(narray, i));
777                                        }
778                                }
779                        }
780                        return o;
781                }
782
783                @Override
784                public Class<? extends Annotation> getAnnClass() {
785                        return Transposable.class;
786                }
787
788                @Override
789                public int change(int axis) {
790                        return 0;
791                }
792
793                @Override
794                public int getNewRank() {
795                        return -1;
796                }
797
798                @Override
799                public ILazyDataset run(ILazyDataset lz) {
800                        return lz.getTransposedView(map);
801                }
802        }
803
804        class MdsDirty implements MetadatasetAnnotationOperation {
805
806                @Override
807                public Object processField(Field f, Object o) {
808                        // throw exception if not boolean???
809                        Class<?> t = f.getType();
810                        if (t.equals(boolean.class) || t.equals(Boolean.class)) {
811                                if (o.equals(false)) {
812                                        o = true;
813                                }
814                        }
815                        return o;
816                }
817
818                @Override
819                public Class<? extends Annotation> getAnnClass() {
820                        return Dirtiable.class;
821                }
822
823                @Override
824                public int change(int axis) {
825                        return 0;
826                }
827
828                @Override
829                public int getNewRank() {
830                        return -1;
831                }
832
833                @Override
834                public ILazyDataset run(ILazyDataset lz) {
835                        return lz;
836                }
837        }
838
839        /**
840         * Slice all datasets in metadata that are annotated by @Sliceable. Call this on the new sliced
841         * dataset after cloning the metadata
842         * @param asView if true then just a view
843         * @param slice
844         */
845        protected void sliceMetadata(boolean asView, final SliceND slice) {
846                processAnnotatedMetadata(new MdsSlice(asView, slice), true);
847        }
848
849        /**
850         * Reshape all datasets in metadata that are annotated by @Reshapeable. Call this when squeezing
851         * or setting the shape
852         * 
853         * @param newShape
854         */
855        protected void reshapeMetadata(final int[] oldShape, final int[] newShape) {
856                processAnnotatedMetadata(new MdsReshape(oldShape, newShape), true);
857        }
858
859        /**
860         * Transpose all datasets in metadata that are annotated by @Transposable. Call this on the transposed
861         * dataset after cloning the metadata
862         * @param axesMap
863         */
864        protected void transposeMetadata(final int[] axesMap) {
865                processAnnotatedMetadata(new MdsTranspose(axesMap), true);
866        }
867
868        /**
869         * Dirty metadata that are annotated by @Dirtiable. Call this when the dataset has been modified
870         * @since 2.0
871         */
872        protected void dirtyMetadata() {
873                processAnnotatedMetadata(new MdsDirty(), true);
874        }
875
876        @SuppressWarnings("unchecked")
877        private void processAnnotatedMetadata(MetadatasetAnnotationOperation op, boolean throwException) {
878                if (metadata == null)
879                        return;
880
881                for (List<MetadataType> l : metadata.values()) {
882                        for (MetadataType m : l) {
883                                if (m == null) {
884                                        continue;
885                                }
886
887                                Class<? extends MetadataType> mc = m.getClass();
888                                do { // iterate over super-classes
889                                        processClass(op, m, mc, throwException);
890                                        Class<?> sclazz = mc.getSuperclass();
891                                        if (!MetadataType.class.isAssignableFrom(sclazz)) {
892                                                break;
893                                        }
894                                        mc = (Class<? extends MetadataType>) sclazz;
895                                } while (true);
896                        }
897                }
898        }
899
900        @SuppressWarnings({ "unchecked", "rawtypes" })
901        private static void processClass(MetadatasetAnnotationOperation op, MetadataType m, Class<? extends MetadataType> mc, boolean throwException) {
902                for (Field f : mc.getDeclaredFields()) {
903                        if (!f.isAnnotationPresent(op.getAnnClass()))
904                                continue;
905
906                        try {
907                                f.setAccessible(true);
908                                Object o = f.get(m);
909                                if (o == null) {
910                                        continue;
911                                }
912
913                                Object no = op.processField(f, o);
914                                if (no != o) {
915                                        f.set(m, no);
916                                        continue;
917                                }
918                                Object r = null;
919                                if (o instanceof ILazyDataset) {
920                                        try {
921                                                f.set(m, op.run((ILazyDataset) o));
922                                        } catch (Exception e) {
923                                                logger.error("Problem processing " + o, e);
924                                                if (!catchExceptions) {
925                                                        throw e;
926                                                }
927                                        }
928                                } else if (o.getClass().isArray()) {
929                                        int l = Array.getLength(o);
930                                        if (l <= 0) {
931                                                continue;
932                                        }
933
934                                        for (int i = 0; r == null && i < l; i++) {
935                                                r = Array.get(o, i);
936                                        }
937                                        int n = op.getNewRank();
938                                        if (r == null) {
939                                                if (n < 0 || n != l) { // all nulls be need to match rank as necessary
940                                                        f.set(m, Array.newInstance(o.getClass().getComponentType(), n < 0 ? l : n));
941                                                }
942                                                continue;
943                                        }
944                                        if (n < 0) {
945                                                n = l;
946                                        }
947                                        Object narray = Array.newInstance(r.getClass(), n);
948                                        for (int i = 0, si = 0, di = 0; di < n && si < l; i++) {
949                                                int c = op.change(i);
950                                                if (c == 0) {
951                                                        Array.set(narray, di++, processObject(op, Array.get(o, si++)));
952                                                } else if (c > 0) {
953                                                        di += c; // add nulls by skipping forward in destination array
954                                                } else if (c < 0) {
955                                                        si -= c; // remove dimensions by skipping forward in source array
956                                                }
957                                        }
958                                        if (n == l) {
959                                                for (int i = 0; i < l; i++) {
960                                                        Array.set(o, i, Array.get(narray, i));
961                                                }
962                                        } else {
963                                                f.set(m, narray);
964                                        }
965                                } else if (o instanceof List<?>) {
966                                        List list = (List) o;
967                                        int l = list.size();
968                                        if (l <= 0) {
969                                                continue;
970                                        }
971
972                                        for (int i = 0; r == null && i < l; i++) {
973                                                r = list.get(i);
974                                        }
975                                        int n = op.getNewRank();
976                                        if (r == null) {
977                                                if (n < 0 || n != l) { // all nulls be need to match rank as necessary
978                                                        list.clear();
979                                                        for (int i = 0, imax = n < 0 ? l : n; i < imax; i++) {
980                                                                list.add(null);
981                                                        }
982                                                }
983                                                continue;
984                                        }
985
986                                        if (n < 0) {
987                                                n = l;
988                                        }
989                                        Object narray = Array.newInstance(r.getClass(), n);
990                                        for (int i = 0, si = 0, di = 0; i < l && si < l; i++) {
991                                                int c = op.change(i);
992                                                if (c == 0) {
993                                                        Array.set(narray, di++, processObject(op, list.get(si++)));
994                                                } else if (c > 0) {
995                                                        di += c; // add nulls by skipping forward in destination array
996                                                } else if (c < 0) {
997                                                        si -= c; // remove dimensions by skipping forward in source array
998                                                }
999                                        }
1000                                        list.clear();
1001                                        for (int i = 0; i < n; i++) {
1002                                                list.add(Array.get(narray, i));
1003                                        }
1004                                } else if (o instanceof Map<?,?>) {
1005                                        Map map = (Map) o;
1006                                        for (Object k : map.keySet()) {
1007                                                map.put(k, processObject(op, map.get(k)));
1008                                        }
1009                                }
1010                        } catch (Exception e) {
1011                                logger.error("Problem occurred when processing metadata of class {}: {}", mc.getCanonicalName(), e);
1012                                if (throwException) {
1013                                        throw new RuntimeException(e);
1014                                } 
1015                        }
1016                }
1017        }
1018
1019        @SuppressWarnings({ "unchecked", "rawtypes" })
1020        private static Object processObject(MetadatasetAnnotationOperation op, Object o) throws Exception {
1021                if (o == null) {
1022                        return o;
1023                }
1024
1025                if (o instanceof ILazyDataset) {
1026                        try {
1027                                return op.run((ILazyDataset) o);
1028                        } catch (Exception e) {
1029                                logger.error("Problem processing " + o, e);
1030                                if (!catchExceptions) {
1031                                        throw e;
1032                                }
1033                        }
1034                } else if (o.getClass().isArray()) {
1035                        int l = Array.getLength(o);
1036                        for (int i = 0; i < l; i++) {
1037                                Array.set(o, i, processObject(op, Array.get(o, i)));
1038                        }
1039                } else if (o instanceof List<?>) {
1040                        List list = (List) o;
1041                        for (int i = 0, imax = list.size(); i < imax; i++) {
1042                                list.set(i, processObject(op, list.get(i)));
1043                        }
1044                } else if (o instanceof Map<?,?>) {
1045                        Map map = (Map) o;
1046                        for (Object k : map.keySet()) {
1047                                map.put(k, processObject(op, map.get(k)));
1048                        }
1049                }
1050                return o;
1051        }
1052
1053        protected ILazyDataset createFromSerializable(Serializable blob, boolean keepLazy) {
1054                ILazyDataset d = null;
1055                if (blob instanceof ILazyDataset) {
1056                        d = (ILazyDataset) blob;
1057                        if (d instanceof IDataset) {
1058                                Dataset ed = DatasetUtils.convertToDataset((IDataset) d);
1059                                int is = ed.getElementsPerItem();
1060                                if (is != 1 && is != getElementsPerItem()) {
1061                                        throw new IllegalArgumentException("Dataset has incompatible number of elements with this dataset");
1062                                }
1063                                d = ed.cast(is == 1 ? Dataset.FLOAT64 : Dataset.ARRAYFLOAT64);
1064                        } else if (!keepLazy) {
1065                                final int is = getElementsPerItem();
1066                                try {
1067                                        d = DatasetUtils.cast(d.getSlice(), is == 1 ? Dataset.FLOAT64 : Dataset.ARRAYFLOAT64);
1068                                } catch (DatasetException e) {
1069                                        logger.error("Could not get data from lazy dataset", e);
1070                                        return null;
1071                                }
1072                        }
1073                } else {
1074                        final int is = getElementsPerItem();
1075                        if (is == 1) {
1076                                d = DatasetFactory.createFromObject(DoubleDataset.class, blob);
1077                        } else {
1078                                try {
1079                                        d = DatasetFactory.createFromObject(is, CompoundDoubleDataset.class, blob);
1080                                } catch (IllegalArgumentException e) { // if only single value supplied try again
1081                                        d = DatasetFactory.createFromObject(DoubleDataset.class, blob);
1082                                }
1083                        }
1084                        if (d.getSize() == getSize() && !Arrays.equals(d.getShape(), shape)) {
1085                                d.setShape(shape.clone());
1086                        }
1087                }
1088                List<int[]> s = BroadcastUtils.broadcastShapesToMax(shape, d.getShape());
1089                d.setShape(s.get(0));
1090
1091                return d;
1092        }
1093
1094        @Override
1095        public void setErrors(Serializable errors) {
1096                if (shape == null) {
1097                        throw new IllegalArgumentException("Cannot set errors for null dataset");
1098                }
1099                if (errors == null) {
1100                        clearMetadata(ErrorMetadata.class);
1101                        return;
1102                }
1103                if (errors == this) {
1104                        logger.warn("Ignoring setting error to itself as this will lead to infinite recursion");
1105                        return;
1106                }
1107
1108                ILazyDataset errorData = createFromSerializable(errors, true);
1109
1110                ErrorMetadata emd = getErrorMetadata();
1111                if (emd == null) {
1112                        try {
1113                                emd = MetadataFactory.createMetadata(ErrorMetadata.class);
1114                                setMetadata(emd);
1115                        } catch (MetadataException me) {
1116                                logger.error("Could not create metadata", me);
1117                        }
1118                }
1119                emd.setError(errorData);
1120        }
1121
1122        protected ErrorMetadata getErrorMetadata() {
1123                try {
1124                        List<ErrorMetadata> el = getMetadata(ErrorMetadata.class);
1125                        if (el != null && !el.isEmpty()) {
1126                                 return el.get(0);
1127                        }
1128                } catch (Exception e) {
1129                }
1130                return null;
1131        }
1132
1133        @Override
1134        public ILazyDataset getErrors() {
1135                ErrorMetadata emd = getErrorMetadata();
1136                return emd == null ? null : emd.getError();
1137        }
1138
1139        @Override
1140        public boolean hasErrors() {
1141                return LazyDatasetBase.this.getErrors() != null;
1142        }
1143
1144        /**
1145         * Check permutation axes
1146         * @param shape
1147         * @param axes
1148         * @return cleaned up axes or null if trivial
1149         */
1150        public static int[] checkPermutatedAxes(int[] shape, int... axes) {
1151                int rank = shape == null ? 0 : shape.length;
1152
1153                if (axes == null || axes.length == 0) {
1154                        axes = new int[rank];
1155                        for (int i = 0; i < rank; i++) {
1156                                axes[i] = rank - 1 - i;
1157                        }
1158                }
1159
1160                if (axes.length != rank) {
1161                        logger.error("axis permutation has length {} that does not match dataset's rank {}", axes.length, rank);
1162                        throw new IllegalArgumentException("axis permutation does not match shape of dataset");
1163                }
1164        
1165                // check all permutation values are within bounds
1166                for (int i = 0; i < rank; i++) {
1167                        axes[i] = ShapeUtils.checkAxis(rank, axes[i]);
1168                }
1169        
1170                // check for a valid permutation (is this an unnecessary restriction?)
1171                int[] perm = axes.clone();
1172                Arrays.sort(perm);
1173
1174                for (int i = 0; i < rank; i++) {
1175                        if (perm[i] != i) {
1176                                logger.error("axis permutation is not valid: it does not contain complete set of axes");
1177                                throw new IllegalArgumentException("axis permutation does not contain complete set of axes");
1178                        }
1179                }
1180
1181                if (Arrays.equals(axes, perm)) {
1182                        return null; // signal identity or trivial permutation
1183                }
1184
1185                return axes;
1186        }
1187}