fix signed integer overflow in scan_scaled. Found by Nicolas Iooss

using AFL against ssh_config. ok deraadt@ millert@

src/lib/libutil/fmt_scaled.c

@@ -1,4 +1,4 @@
-/*	$OpenBSD: fmt_scaled.c,v 1.12 2013/11/29 19:00:51 deraadt Exp $	*/
+/*	$OpenBSD: fmt_scaled.c,v 1.13 2017/03/11 23:37:23 djm Exp $	*/
 
 /*
  * Copyright (c) 2001, 2002, 2003 Ian F. Darwin.  All rights reserved.
@@ -121,6 +121,10 @@ scan_scaled(char *scaled, long long *result)
 				/* ignore extra fractional digits */
 				continue;
 			fract_digits++;		/* for later scaling */
+			if (fpart >= LLONG_MAX / 10) {
+				errno = ERANGE;
+				return -1;
+			}
 			fpart *= 10;
 			fpart += i;
 		} else {				/* normal digit */
@@ -128,6 +132,10 @@ scan_scaled(char *scaled, long long *result)
 				errno = ERANGE;
 				return -1;
 			}
+			if (whole >= LLONG_MAX / 10) {
+				errno = ERANGE;
+				return -1;
+			}
 			whole *= 10;
 			whole += i;
 		}
@@ -158,6 +166,11 @@ scan_scaled(char *scaled, long long *result)
 			}
 			scale_fact = scale_factors[i];
 
+			if (whole >= LLONG_MAX / scale_fact) {
+				errno = ERANGE;
+				return -1;
+			}
+
 			/* scale whole part */
 			whole *= scale_fact;